<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html>
<head>
 
 <title>papy - Google Code</title>
 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
 
 <link rel="shortcut icon" href="http://www.gstatic.com/codesite/ph/images/defaultlogo.png"/>
 <link type="text/css" rel="stylesheet"
       href="./_static/custom.css" />
 <link type="text/css" rel="stylesheet"
       href="http://www.gstatic.com/codesite/ph/3770121226765569314/css/ph_core.css" />
 <link type="text/css" rel="stylesheet"
       href="http://www.gstatic.com/codesite/ph/3770121226765569314/css/ph_detail.css" />
 <link rel="stylesheet"  type="text/css"
       href="./_static/pygments.css" /> 
 
 <link type="application/atom+xml" rel="alternate"
 href="/feeds/p/papy/svnchanges/basic?path=/wiki/" />
 
 
<!--[if IE]>
 <link type="text/css" rel="stylesheet"
       href="http://code.google.com/hosting/css/d_ie.css" />
<![endif]-->

<script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
          URL_ROOT:    '',
          VERSION:     '1.0b1',
          COLLAPSE_MODINDEX: false,
          FILE_SUFFIX: '.html'
      };
    </script>
    <script type="text/javascript" src="./_static/jquery.js"></script>
    <script type="text/javascript" src="./_static/interface.js"></script>
    <script type="text/javascript" src="./_static/doctools.js"></script>
    <link rel="contents" title="Global table of contents" href="contents.html" />
    <link rel="index" title="Global index" href="genindex.html" />
    <link rel="search" title="Search" href="search.html" />
    <link rel="top" title="papy v1.0b1 documentation" href="index.html" />
    <link rel="prev" title="TODO" href="todo.html" />
</head>
<body class="t6">
 <div id=gaia>
  <font size="-1">
 
 <a href="http://code.google.com/p/support/wiki/WhatsNew" style="color:#a03">What's new?</a>
 | <a href="http://code.google.com/p/support/">Help</a>
 
 </font> 

 </div>
 <table style="padding:0px; margin: 0px 0px -6px 0; width:100%" cellpadding=0 cellspacing=0>
 <tr>
 <td><a href="/"><img src="http://www.gstatic.com/codesite/ph/images/defaultlogo.png" alt=papy/></a></td>
 <td>
 
 <div id="pname">
 <a href="http://code.google.com/p/papy/" style="text-decoration:none; color:#000">papy</a>
 
 </div>
 <div id="psum">
 <i><a href="http://code.google.com/p/papy/" style="text-decoration:none; color:#000">papy v1.0b1 documentation</a></i>
 </div>
 
 </td>
 <td style="white-space:nowrap; text-align:right">
 
 <form action="http://code.google.com/hosting/search">
 <input size=30 name=q value=""/>
 <input type=submit name=projectsearch value="Search Projects" />
 <input type=submit name=websearch value="Search the Web" />
 </form>
 
 </tr>
 </table>


<table id="mt" cellspacing="0" cellpadding="0" width="100%" border="0">
 <tr>
 <th onclick="if (!cancelBubble) _go('http://code.google.com/p/papy/');">
 <div class="tab inactive">
 <div class="round4"></div>
 <div class="round2"></div>
 <div class="round1"></div>
 <div class="box-inner">
 <a onclick="cancelBubble=true;" href="http://code.google.com/p/papy/">Project&nbsp;Home</a>
 </div>
 </div>
 </th><td>&nbsp;&nbsp;</td>
 
 
 
 
 <th onclick="if (!cancelBubble) _go('http://code.google.com/p/papy/downloads/list');">
 <div class="tab inactive">
 <div class="round4"></div>
 <div class="round2"></div>
 <div class="round1"></div>
 <div class="box-inner">
 <a onclick="cancelBubble=true;" href="http://code.google.com/p/papy/downloads/list">Downloads</a>
 </div>
 </div>
 </th><td>&nbsp;&nbsp;</td>
 
 
 
 
 
 <th onclick="if (!cancelBubble) _go('http://code.google.com/p/papy/w/list');">
 <div class="tab inactive">
 <div class="round4"></div>
 <div class="round2"></div>
 <div class="round1"></div>
 <div class="box-inner">
 <a onclick="cancelBubble=true;" href="http://code.google.com/p/papy/w/list">Wiki</a>
 </div>
 </div>
 </th><td>&nbsp;&nbsp;</td>
 
 
 
 
 
 
 
 <th onclick="if (!cancelBubble) _go('http://code.google.com/p/papy/issues/list');">
 <div class="tab inactive">
 <div class="round4"></div>
 <div class="round2"></div>
 <div class="round1"></div>
 <div class="box-inner">
 <a onclick="cancelBubble=true;" href="http://code.google.com/p/papy/issues/list">Issues</a>
 </div>
 </div>
 </th><td>&nbsp;&nbsp;</td>
 
 
 
 
 
 
 <th onclick="if (!cancelBubble) _go('http://code.google.com/p/papy/source/list');">
 <div class="tab inactive">
 <div class="round4"></div>
 <div class="round2"></div>
 <div class="round1"></div>
 <div class="box-inner">
 <a onclick="cancelBubble=true;" href="http://code.google.com/p/papy/source/list">Source</a>
 </div>
 </div>
 </th><td>&nbsp;&nbsp;</td>
 
 
 
 <td width="100%">&nbsp;</td>
 </tr>
</table>
<table cellspacing="0" cellpadding="0" width="100%" align="center" border="0" class="st">
 <tr>
 
 
 
 <td>
 <div class="issueList">
<div class=isf>
 
 
 
 <a href="index.html">papy v1.0b1 documentation</a> |
 <span>
   <a href="genindex.html" title="General Index"
      accesskey="I">index</a>
 </span> |
 <span>
   <a href="modindex.html" title="Global Module Index"
      accesskey="M">modules</a>
 </span> |
 <span>
   <a href="todo.html" title="TODO"
      accesskey="P">previous</a>
 </span> |
 <span class="inIssueList">
   <span>Search</span>
    <form action="search.html" method="GET" style="display:inline">
      <input size=30 name=q value=""/>
      <input type=submit name=projectsearch value="Search Documentation" />
      <input type="hidden" name="check_keywords" value="yes" />
      <input type="hidden" name="area" value="default" />
    </form>
 </span>

</div>
</div>

 </td>
 
 
 
 
 
 
 
 
 <td height=4 align=right valign=top class="bevel-right">
 <div class="round4"></div>
 <div class="round2"></div>
 <div class="round1"></div>
 </td>
 </tr>
</table>
<script>
 var cancelBubble = false;
 function _go(url) { document.location = url; }
</script>

<div id="maincol">
<!-- IE -->



<style>
 #downloadbox { 
 padding: 6px; 
 overflow: hidden;
 white-space: nowrap;
 text-overflow: ellipsis;
 }
 #downloadbox a {
 margin: 0 0 0 0;
 display: block;
 padding-left: 0px;
 }
 #owners a, #members a { white-space: nowrap; }
</style>

<div style="float:right; width:25em; margin:0 0 1em 2em">

 
 <div class=pmeta_bubble_bg>
 <div class="round4"></div>
 <div class="round2"></div>

 <div class="round1"></div>
 <div class="box-inner">
 <table class="pmeta" cellpadding=5>
 <tr><th>Previous:</th>
 <td><ul>
 
 <li>
   <a href="todo.html" title="previous chapter">TODO</a>
 </li>
 

 
 </ul></td>
 </tr>
 
 
 
 </table>

 </div>
 <div class="round1"></div>
 <div class="round2"></div>
 <div class="round4"></div>
 </div>
 
 
 
 <div style="background:#ddf8cc; margin-bottom: 5px;table-layout:fixed">

 <div class="round4"></div>

 <div class="round2"></div>
 <div class="round1"></div>
 <div class="box-inner">
 <div id=downloadbox>
  <b>Table of Contents</b><br/>
  <table class="pmeta toc" cellpadding=5><tr><td>
	<ul>
<li><a class="reference external" href=""><tt class="docutils literal"><span class="pre">papy.papy</span></tt></a></li>
<li><a class="reference external" href="#papy-graph"><tt class="docutils literal"><span class="pre">papy.graph</span></tt></a></li>
<li><a class="reference external" href="#papy-utils"><tt class="docutils literal"><span class="pre">papy.utils</span></tt></a></li>
<li><a class="reference external" href="#papy-tkgui"><tt class="docutils literal"><span class="pre">papy.tkgui</span></tt></a></li>
<li><a class="reference external" href="#papy-workers"><tt class="docutils literal"><span class="pre">papy.workers</span></tt></a></li>
<li><a class="reference external" href="#papy-workers-core"><tt class="docutils literal"><span class="pre">papy.workers.core</span></tt></a></li>
<li><a class="reference external" href="#papy-workers-maths"><tt class="docutils literal"><span class="pre">papy.workers.maths</span></tt></a></li>
<li><a class="reference external" href="#papy-workers-io"><tt class="docutils literal"><span class="pre">papy.workers.io</span></tt></a></li>
<li><a class="reference external" href="#imap-imap"><tt class="docutils literal"><span class="pre">IMap.IMap</span></tt></a></li>
</ul>

  </td></tr></table>

 
 </div>
 </div>
 <div class="round1"></div>
 <div class="round2"></div>
 <div class="round4"></div>
 </div>
 
 


 </div>
</div>

 <div style="padding:0 3em 1.2em 0">
   
     
  <div class="section" id="papy-papy">
<span id="module-papy.papy"></span><h1><tt class="xref docutils literal"><span class="pre">papy.papy</span></tt><a class="headerlink" href="#papy-papy" title="Permalink to this headline">¶</a></h1>
<p>This module provides classes and functions to construct and run a <em>PaPy</em> 
pipeline.</p>
<dl class="class">
<dt id="papy.papy.Chain">
<em class="property">class </em><tt class="descclassname">papy.papy.</tt><tt class="descname">Chain</tt><big>(</big><em>iterables</em>, <em>stride=1</em><big>)</big><a class="headerlink" href="#papy.papy.Chain" title="Permalink to this definition">¶</a></dt>
<dd><p>This is a generalization of the <tt class="docutils literal"><span class="pre">zip</span></tt> and <tt class="docutils literal"><span class="pre">chain</span></tt> functions. 
If stride =1 it behaves like <tt class="docutils literal"><span class="pre">itertools.zip</span></tt>, if stride =len(iterable) it
behaves like <tt class="docutils literal"><span class="pre">itertools.chain</span></tt> in any other case it zips iterables in 
strides e.g:</p>
<div class="highlight-python"><pre>a = Chain([iter([1,2,3]), iter([4,5,6], stride =2)
list(a)
&gt;&gt;&gt; [1,2,4,5,3,6]</pre>
</div>
<p>It is further resistant to exceptions i.e. if one of the iterables
raises an exception the <tt class="docutils literal"><span class="pre">Chain</span></tt> does not end in a <tt class="docutils literal"><span class="pre">StopIteration</span></tt>, but 
continues with other iterables.</p>
<dl class="method">
<dt id="papy.papy.Chain.next">
<tt class="descname">next</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.papy.Chain.next" title="Permalink to this definition">¶</a></dt>
<dd>Returns the next result from the chained iterables given stride.</dd></dl>

</dd></dl>

<dl class="class">
<dt id="papy.papy.Consume">
<em class="property">class </em><tt class="descclassname">papy.papy.</tt><tt class="descname">Consume</tt><big>(</big><em>iterable</em>, <em>n=1</em>, <em>stride=1</em><big>)</big><a class="headerlink" href="#papy.papy.Consume" title="Permalink to this definition">¶</a></dt>
<dd><p>This iterator-wrapper consumes n results from the input iterator and weaves
the results together in strides. If the result is an exception it is <em>not</em>
raised.</p>
<dl class="method">
<dt id="papy.papy.Consume.next">
<tt class="descname">next</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.papy.Consume.next" title="Permalink to this definition">¶</a></dt>
<dd>Returns the next sequence of results, given stride and n.</dd></dl>

</dd></dl>

<dl class="class">
<dt id="papy.papy.Dagger">
<em class="property">class </em><tt class="descclassname">papy.papy.</tt><tt class="descname">Dagger</tt><big>(</big><em>pipers=()</em>, <em>pipes=()</em>, <em>xtras=None</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger" title="Permalink to this definition">¶</a></dt>
<dd><p>The <em>Dagger</em> is a directed acyclic graph. It defines the topology of a 
<em>PaPy</em> pipeline/workflow. It is a subclass from <em>Graph</em> inverting the 
direction of edges called within the <em>Dagger</em> pipes. Edges can be regarded 
as dependencies, while pipes as data-flow between <em>Pipers</em> or <em>Nodes</em> of the 
<em>Graph</em>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">pipers(sequence) [default: <tt class="docutils literal"><span class="pre">()</span></tt>]</p>
<blockquote>
<p>A sequence of valid <tt class="docutils literal"><span class="pre">add_piper</span></tt> inputs (see the documentation for 
the <tt class="docutils literal"><span class="pre">add_piper</span></tt> method).</p>
</blockquote>
</li>
<li><p class="first">pipes(sequence) [default: <tt class="docutils literal"><span class="pre">()</span></tt>]</p>
<blockquote>
<p>A sequence of valid <tt class="docutils literal"><span class="pre">add_pipe</span></tt> inputs  (see the documentation for 
the <tt class="docutils literal"><span class="pre">add_piper</span></tt> method).</p>
</blockquote>
</li>
</ul>
</blockquote>
<dl class="method">
<dt id="papy.papy.Dagger.add_pipe">
<tt class="descname">add_pipe</tt><big>(</big><em>pipe</em>, <em>branch=None</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.add_pipe" title="Permalink to this definition">¶</a></dt>
<dd><p>Adds a pipe (A, ..., N) which is an N-tuple tuple of <em>Pipers</em>. Adding a 
pipe means to add all the <em>Pipers</em> and connect them in the specified 
left to right order.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">pipe(sequence)</p>
<blockquote>
<p>N-tuple of <em>Piper</em> instances or objects which are valid 
<tt class="docutils literal"><span class="pre">add_piper</span></tt> arguments. See: <tt class="docutils literal"><span class="pre">Dagger.add_piper</span></tt> and 
<tt class="docutils literal"><span class="pre">Dagger.resolve</span></tt>.</p>
</blockquote>
</li>
</ul>
</blockquote>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">The direction of the edges in the graph is reversed compared to the
left to right data-flow in a pipe.</p>
</div>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.add_piper">
<tt class="descname">add_piper</tt><big>(</big><em>piper</em>, <em>xtra=None</em>, <em>create=True</em>, <em>branch=None</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.add_piper" title="Permalink to this definition">¶</a></dt>
<dd><p>Adds a <em>Piper</em> to the <em>Dagger</em>, only if the <em>Piper</em> is not already in 
a <em>Node</em>. Optionally creates a new <em>Piper</em> if the piper argument is 
valid for the <em>Piper</em> constructor. Returns a tuple: (new_piper_created,
piper_instance) indicating whether a new <em>Piper</em> has been created
and the instance of the added <em>Piper</em>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">piper(<em>Piper</em> instance, <em>Worker</em> instance or id(<em>Piper</em> instance))</p>
<blockquote>
<p><em>Piper</em> instance or object which will be converted to a <em>Piper</em> 
instance.</p>
</blockquote>
</li>
<li><p class="first">create(bool) [default: <tt class="xref docutils literal"><span class="pre">True</span></tt>]</p>
<blockquote>
<p>Should a new <em>Piper</em> be created if the piper cannot be resolved 
in the <em>Dagger</em>?</p>
</blockquote>
</li>
<li><p class="first">xtra(dict) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
<blockquote>
<p>Dictionary of <em>Graph</em> <em>Node</em> properties.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.add_pipers">
<tt class="descname">add_pipers</tt><big>(</big><em>pipers</em>, <em>*args</em>, <em>**kwargs</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.add_pipers" title="Permalink to this definition">¶</a></dt>
<dd><p>Adds a sequence of <em>Pipers</em> to the <em>Dagger</em> in specified order. Takes 
optional arguments for <tt class="docutils literal"><span class="pre">Dagger.add_piper</span></tt>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">pipers (sequence of valid <tt class="docutils literal"><span class="pre">add_piper</span></tt> arguments)</p>
<blockquote>
<p>Sequence of <em>Pipers</em> or valid <tt class="docutils literal"><span class="pre">Dagger.add_piper</span></tt> arguments 
to be added to the <em>Dagger</em> in the left to right order of the 
sequence.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.add_pipes">
<tt class="descname">add_pipes</tt><big>(</big><em>pipes</em>, <em>*args</em>, <em>**kwargs</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.add_pipes" title="Permalink to this definition">¶</a></dt>
<dd><p>Adds a sequence of pipes to the <em>Dagger</em> in the specified order. 
Takes optional arguments for <tt class="docutils literal"><span class="pre">Dagger.add_pipe</span></tt>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">pipes (sequence of valid <tt class="docutils literal"><span class="pre">add_pipe</span></tt> arguments)</p>
<blockquote>
<p>Sequence of pipes or valid <tt class="docutils literal"><span class="pre">Dagger.add_pipe</span></tt> arguments to be 
added to the <em>Dagger</em> in the left to right order of the 
sequence.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.children_after_parents">
<tt class="descname">children_after_parents</tt><big>(</big><em>piper1</em>, <em>piper2</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.children_after_parents" title="Permalink to this definition">¶</a></dt>
<dd>Custom compare function. Returns 1 if the first Piper is upstream of the
second Piper, -1 if the first Piper is downstream of the second Piper 
and 0 if the two Pipers are independent.</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.connect">
<tt class="descname">connect</tt><big>(</big><em>datas=None</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.connect" title="Permalink to this definition">¶</a></dt>
<dd>Given the pipeline topology connects <em>Pipers</em> in the order input -&gt; 
output. See <tt class="docutils literal"><span class="pre">Piper.connect</span></tt>.</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.connect_inputs">
<tt class="descname">connect_inputs</tt><big>(</big><em>datas</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.connect_inputs" title="Permalink to this definition">¶</a></dt>
<dd><p>Connects input <em>Pipers</em> to input data in the correct order determined,
by the <tt class="docutils literal"><span class="pre">Piper.ornament</span></tt> attribute and the <tt class="docutils literal"><span class="pre">Dagger._cmp</span></tt> function.</p>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">It is assumed that the input data is in the form of an iterator and
that all inputs have the same number of input items. A pipeline will
deadlock otherwise.</p>
</div>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">datas (sequence of iterators)</p>
<blockquote>
<p>Ordered sequence of inputs for all input <em>Pipers</em>.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.del_pipe">
<tt class="descname">del_pipe</tt><big>(</big><em>pipe</em>, <em>forced=False</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.del_pipe" title="Permalink to this definition">¶</a></dt>
<dd><p>Deletes a pipe (A, ..., N) which is an N-tuple of pipers. Deleting a 
pipe means to delete all the connections between pipers and to delete
all the <em>Pipers</em>. If forced =``False`` only <em>Pipers</em> which are not 
needed anymore (i.e. have not downstream <em>Pipers</em>) are deleted.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">pipe(sequence)</p>
<blockquote>
<p>N-tuple of <em>Piper</em> instances or objects which can be resolved in 
the <em>Dagger</em> (see: <tt class="docutils literal"><span class="pre">Dagger.resolve</span></tt>). The <em>Pipers</em> are removed
from right to left.</p>
</blockquote>
</li>
<li><p class="first">forced(bool) [default: <tt class="xref docutils literal"><span class="pre">False</span></tt>]</p>
<blockquote>
<p>The forced argument will be forwarded to the <tt class="docutils literal"><span class="pre">Dagger.del_piper</span></tt>
method. If forced is <tt class="xref docutils literal"><span class="pre">False</span></tt> only <em>Pipers</em> with no outgoing 
pipes will be deleted.</p>
</blockquote>
</li>
</ul>
</blockquote>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">The direction of the edges in the <em>Graph</em> is reversed compared to 
the  left to right data-flow in a pipe.</p>
</div>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.del_piper">
<tt class="descname">del_piper</tt><big>(</big><em>piper</em>, <em>forced=False</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.del_piper" title="Permalink to this definition">¶</a></dt>
<dd><p>Removes a <em>Piper</em> from the <em>Dagger</em>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">piper(<em>Piper</em> instance, <em>Worker</em> instance or id(<em>Piper</em> instance))</p>
<blockquote>
<p><em>Piper</em> instance or object from which a <em>Piper</em> instance can 
be constructed.</p>
</blockquote>
</li>
<li><p class="first">forced(bool) [default: <tt class="xref docutils literal"><span class="pre">False</span></tt>]</p>
<blockquote>
<p>If forced =``False <em>Pipers</em> with outgoing pipes (incoming 
edges) will not be removed and will raise a <tt class="docutils literal"><span class="pre">DaggerError</span></tt>.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.del_pipers">
<tt class="descname">del_pipers</tt><big>(</big><em>pipers</em>, <em>*args</em>, <em>**kwargs</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.del_pipers" title="Permalink to this definition">¶</a></dt>
<dd><p>Deletes a sequence of <em>Pipers</em> from the <em>Dagger</em> in reverse of the 
specified order. Takes optional arguments for <tt class="docutils literal"><span class="pre">Dagger.del_piper</span></tt>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">pipes (sequence of valid <tt class="docutils literal"><span class="pre">del_pipe</span></tt> arguments)</p>
<blockquote>
<p>Sequence of <em>Pipers</em> or valid <tt class="docutils literal"><span class="pre">Dagger.del_piper</span></tt> arguments 
to be removed from the <em>Dagger</em> in the right to left order of 
the sequence.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.del_pipes">
<tt class="descname">del_pipes</tt><big>(</big><em>pipes</em>, <em>*args</em>, <em>**kwargs</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.del_pipes" title="Permalink to this definition">¶</a></dt>
<dd><p>Deletes a sequence of pipes from the <em>Dagger</em> in the specified order. 
Takes optional arguments for <tt class="docutils literal"><span class="pre">Dagger.del_pipe</span></tt>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">pipes (sequence of valid <tt class="docutils literal"><span class="pre">del_pipe</span></tt> arguments)</p>
<blockquote>
<p>Sequence of pipes or valid <tt class="docutils literal"><span class="pre">Dagger.del_pipe</span></tt> arguments to be 
removed from the <em>Dagger</em> in left to right order of the 
sequence.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.disconnect">
<tt class="descname">disconnect</tt><big>(</big><em>forced=False</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.disconnect" title="Permalink to this definition">¶</a></dt>
<dd><p>Given the pipeline topology disconnects <em>Pipers</em> in the order output -&gt; 
input. This also disconnects inputs. See <tt class="docutils literal"><span class="pre">Dagger.connect</span></tt>,
<tt class="docutils literal"><span class="pre">Piper.connect</span></tt> and <tt class="docutils literal"><span class="pre">Piper.disconnect</span></tt>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">forced(bool) [default: False]</p>
<blockquote>
<p>If set <tt class="xref docutils literal"><span class="pre">True</span></tt> all tasks from all IMaps will be removed.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.get_inputs">
<tt class="descname">get_inputs</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.papy.Dagger.get_inputs" title="Permalink to this definition">¶</a></dt>
<dd>Returns <em>Pipers</em> which are inputs to the pipeline i.e. have no 
incoming pipes (outgoing dependency edges).</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.get_outputs">
<tt class="descname">get_outputs</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.papy.Dagger.get_outputs" title="Permalink to this definition">¶</a></dt>
<dd>Returns <em>Pipers</em> which are outputs to the pipeline i.e. have no 
outgoing pipes (incoming dependency edges).</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.resolve">
<tt class="descname">resolve</tt><big>(</big><em>piper</em>, <em>forgive=False</em><big>)</big><a class="headerlink" href="#papy.papy.Dagger.resolve" title="Permalink to this definition">¶</a></dt>
<dd><p>Given a <em>Piper</em> or the <tt class="docutils literal"><span class="pre">id</span></tt> of the <em>Piper</em>. Returns this <em>Piper</em> if it 
can be resolved else raises a <em>DaggerError</em> or returns <tt class="xref docutils literal"><span class="pre">False</span></tt> 
depending on the forgive argument.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">piper(<em>Piper</em> instance, id(<em>Piper</em> instance))</p>
<blockquote>
<p><em>Piper</em> instance or its id to be found in the <em>Dagger</em>.</p>
</blockquote>
</li>
<li><p class="first">forgive(bool) [default =``False``]</p>
<blockquote>
<p>If forgive is <tt class="xref docutils literal"><span class="pre">False</span></tt> a <tt class="docutils literal"><span class="pre">DaggerError</span></tt> is raised whenever a 
<em>Piper</em>  cannot be resolved in the <em>Dagger</em>. If forgive is 
<tt class="xref docutils literal"><span class="pre">True</span></tt>: <tt class="xref docutils literal"><span class="pre">False</span></tt> is returned.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.start">
<tt class="descname">start</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.papy.Dagger.start" title="Permalink to this definition">¶</a></dt>
<dd>Given the pipeline topology starts <em>Pipers</em> in the order input -&gt; 
output. See <tt class="docutils literal"><span class="pre">Piper.start</span></tt>. The forced =`True` argument is passed to 
the <tt class="docutils literal"><span class="pre">Piper.start</span></tt> method, allowing <em>Pipers</em> to share <em>IMaps</em>.</dd></dl>

<dl class="method">
<dt id="papy.papy.Dagger.stop">
<tt class="descname">stop</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.papy.Dagger.stop" title="Permalink to this definition">¶</a></dt>
<dd>Given the pipeline topology stops the <em>Pipers</em>.</dd></dl>

</dd></dl>

<dl class="exception">
<dt id="papy.papy.DaggerError">
<em class="property">exception </em><tt class="descclassname">papy.papy.</tt><tt class="descname">DaggerError</tt><a class="headerlink" href="#papy.papy.DaggerError" title="Permalink to this definition">¶</a></dt>
<dd>Exceptions raised or related to <em>Dagger</em> instances.</dd></dl>

<dl class="class">
<dt id="papy.papy.InputIterator">
<em class="property">class </em><tt class="descclassname">papy.papy.</tt><tt class="descname">InputIterator</tt><big>(</big><em>iterator</em>, <em>piper</em><big>)</big><a class="headerlink" href="#papy.papy.InputIterator" title="Permalink to this definition">¶</a></dt>
<dd><dl class="method">
<dt id="papy.papy.InputIterator.next">
<tt class="descname">next</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.papy.InputIterator.next" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

</dd></dl>

<dl class="class">
<dt id="papy.papy.Piper">
<em class="property">class </em><tt class="descclassname">papy.papy.</tt><tt class="descname">Piper</tt><big>(</big><em>worker</em>, <em>parallel=False</em>, <em>consume=1</em>, <em>produce=1</em>, <em>spawn=1</em>, <em>timeout=None</em>, <em>branch=None</em>, <em>debug=False</em>, <em>name=None</em>, <em>track=False</em><big>)</big><a class="headerlink" href="#papy.papy.Piper" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a new Piper instance.</p>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">The (produce * spawn) of the upstream <em>Piper</em> has to equal the (consume 
* spawn) of the downstream <em>Piper</em>. for each pair of <em>pipers</em> connected
by a pipe. This will be in future enforced by the <em>Dagger</em>.</p>
</div>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">worker(<em>Worker</em> instance, <em>Piper</em> instance or sequence of 
worker-functions or <em>Worker</em> instances)</p>
<blockquote>
<p>A <em>Piper</em> can be created from a <em>Worker</em> instance another <em>Piper</em> 
instance or a sequence of worker-functions or <em>Worker</em> instances
in every case a new instance is created.</p>
</blockquote>
</li>
<li><p class="first">parallel(<tt class="xref docutils literal"><span class="pre">False</span></tt> or <em>IMap</em> instance) [default: <tt class="xref docutils literal"><span class="pre">False</span></tt>]</p>
<blockquote>
<p>If parallel =``False`` <em>Piper</em> will not evaluate the <em>Worker</em> in 
parallel but use the &#8220;manager&#8221; process and the <tt class="docutils literal"><span class="pre">itertools.imap</span></tt> 
function. Otherwise the specified <em>IMap</em> instance and it&#8217;s 
threads/processes will be used.</p>
</blockquote>
</li>
<li><p class="first">consume(int) [default: 1]</p>
<blockquote>
<p>The number of input items consumed from <em>all</em> directly connected 
upstream <em>Pipers</em> per one <em>Worker</em> evaluation. Results will be 
passed to the worker-function as a sequence of individual results.</p>
</blockquote>
</li>
<li><p class="first">produce(int) [default: 1]</p>
<blockquote>
<p>The number of results to generate for each <em>Worker</em> evaluation 
result. Results will be elements of the sequence returned by the
worker.</p>
</blockquote>
</li>
<li><p class="first">spawn(int) [default: 1]</p>
<blockquote>
<p>The number of times this <em>Piper</em> is implicitly added to the pipeline
to consume the specified number of results.</p>
</blockquote>
</li>
<li><p class="first">timeout(int) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
<blockquote>
<p>Time to wait till a result is available. Otherwise a <tt class="docutils literal"><span class="pre">PiperError</span></tt>
is <strong>returned</strong> not raised.</p>
</blockquote>
</li>
<li><p class="first">branch(object) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
<blockquote>
<p>This affects the order of <em>Pipers</em> in the <em>Dagger</em>. <em>Pipers</em> are 
sorted according to:</p>
<blockquote>
<ol class="arabic simple">
<li>data-flow upstream-&gt;downstream</li>
<li>branch attribute</li>
</ol>
</blockquote>
<p>This argument sets the branch attribute of a <em>Piper</em>. If two 
<em>Pipers</em> have no upstream-&gt;downstream relation they will be sorted 
according to their branch attribute. If neither of them has a branch
set or it is identical their order will be semi-random. <em>Pipers</em> 
will implicitly inherit the branch of an up-stream <em>Piper</em>, thus it 
is only necessary to sepcify the branch of a <em>Piper</em> if it is the
first one after a branch point.</p>
<p>The argument can be any object which can be used by the <tt class="docutils literal"><span class="pre">cmp</span></tt> 
built-in function. If necessary they could override the __cmp__ 
method.</p>
<p>Note that it is possible to construct pipelines without specifying 
branches if <em>Pipers</em> which are connected to multiple up-stream 
<em>Pipers</em> use <em>Workers</em> which act correctly regardless of the ordere
of branch results in the inbox of the worker-function.</p>
</blockquote>
</li>
<li><p class="first">debug(bool) [default: False]</p>
<blockquote>
<p>Verbose debugging mode. Raises a <tt class="docutils literal"><span class="pre">PiperError</span></tt> on <tt class="docutils literal"><span class="pre">WorkerErrors</span></tt>.</p>
<div class="admonition warning">
<p class="first admonition-title">Warning</p>
<p class="last">this will most likely hang the Python interpreter after the 
error occurs. Use during development only!</p>
</div>
</blockquote>
</li>
</ul>
</blockquote>
<dl class="method">
<dt id="papy.papy.Piper.connect">
<tt class="descname">connect</tt><big>(</big><em>inbox</em><big>)</big><a class="headerlink" href="#papy.papy.Piper.connect" title="Permalink to this definition">¶</a></dt>
<dd>Connects the <em>Piper</em> to its upstream <em>Pipers</em>. Upstream <em>Pipers</em> should
be passed as a sequence. This connects the <tt class="docutils literal"><span class="pre">Piper.inbox</span></tt> with the 
<tt class="docutils literal"><span class="pre">Piper.outbox</span></tt> respecting the consume, spawn and produce arguments.</dd></dl>

<dl class="method">
<dt id="papy.papy.Piper.disconnect">
<tt class="descname">disconnect</tt><big>(</big><em>forced=False</em><big>)</big><a class="headerlink" href="#papy.papy.Piper.disconnect" title="Permalink to this definition">¶</a></dt>
<dd><p>Disconnects the <em>Piper</em> from its upstream <em>Pipers</em> or input data. If the
<em>Piper</em></p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">forced(bool) [default: <tt class="xref docutils literal"><span class="pre">False</span></tt>]</p>
<blockquote>
<p>If forced is <tt class="xref docutils literal"><span class="pre">True</span></tt> tries to forcefully remove all tasks 
(including the spawned ones) from the <em>IMap</em> instance</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Piper.next">
<tt class="descname">next</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.papy.Piper.next" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns the next result. If no result is availble within the specified 
(at initialization) timeout then a <tt class="docutils literal"><span class="pre">PiperError</span></tt> wrapped TimeoutError 
is returned.</p>
<p>If the result is a <tt class="docutils literal"><span class="pre">WorkerError</span></tt> it is wrapped in a <tt class="docutils literal"><span class="pre">PiperError</span></tt> and 
returned or raised if debug mode was specified at initialization. If 
the result is a <tt class="docutils literal"><span class="pre">PiperError</span></tt> it is propagated.</p>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Piper.start">
<tt class="descname">start</tt><big>(</big><em>stages=None</em><big>)</big><a class="headerlink" href="#papy.papy.Piper.start" title="Permalink to this definition">¶</a></dt>
<dd><p>Makes the <em>Piper</em> ready to return results. This involves starting the 
the provided <em>IMap</em> instance. If multiple <em>Pipers</em> share an <em>IMap</em> 
instance the order in which the <em>Pipers</em> are started is important. The
valid order is upstream before downstream. The <em>IMap</em> instance has to 
be started only once, but the process can be done in 2 stages. 
This methods stages argument is a tuple which can contain any 
combination of the numbers 0, 1 or 2 specifying which stage of the start
routine should be carried out.</p>
<blockquote>
<p>stage 0 - creates the needed <tt class="docutils literal"><span class="pre">itertools.tee</span></tt> objects.</p>
<p>stage 1 - activates <em>IMap</em> pool. A call to next will block..</p>
<p>stage 2 - activates <em>IMap</em> pool managers.</p>
</blockquote>
<p>If this <em>Piper</em> shares an <em>IMap</em> with other <em>Pipers</em> the proper way to
start them is to start them in a valid postorder with 
stages (0, 1) and (2,) separately.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">stages(tuple) [default =(0,) if linear; (0,1,2) if parallel]</p>
<blockquote>
<p>Performs the specified stages of the start of a <em>Piper</em> 
instance. Stage 0  is necessary and sufficient to start a 
<em>Piper</em> utilizing an <tt class="docutils literal"><span class="pre">itertools.imap</span></tt>. Stages 1 and 2 are 
required to start a parallel <em>Piper</em>.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Piper.stop">
<tt class="descname">stop</tt><big>(</big><em>forced=False</em>, <em>**kwargs</em><big>)</big><a class="headerlink" href="#papy.papy.Piper.stop" title="Permalink to this definition">¶</a></dt>
<dd><p>Tries to cleanly stop the <em>Piper</em>. A <em>Piper</em> is &#8220;started&#8221; if it&#8217;s 
<em>IMap</em> instance is &#8220;started&#8221;. Non-parallel <em>Pipers</em> need not to be 
started or stopped. An <em>IMap</em> instance can be stopped by triggering
its stopping procedure and retrieving results from the <em>IMaps</em> end
tasks. Because neither the <em>Piper</em> nor the <em>IMap</em> &#8220;knows&#8221; which 
tasks(<em>Pipers</em>) are the ends they have to be specified:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">end_task_ids</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>    <span class="c"># A list of IMap task ids</span>
<span class="n">piper_instance</span><span class="o">.</span><span class="n">stop</span><span class="p">(</span><span class="n">ends</span> <span class="o">=</span><span class="n">end_task_ids</span><span class="p">)</span>        
</pre></div>
</div>
<p>results in:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">IMap_instance</span><span class="o">.</span><span class="n">stop</span><span class="p">(</span><span class="n">ends</span> <span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">])</span>
</pre></div>
</div>
<p>If the <em>Piper</em> did not finish the forced argument has to be specified:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">piper_instance</span><span class="o">.</span><span class="n">stop</span><span class="p">(</span><span class="n">forced</span> <span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">ends</span> <span class="o">=</span><span class="n">end_task_ids</span><span class="p">)</span>
</pre></div>
</div>
<p>If the <em>Piper</em> (and consequently <em>IMap</em>) is part of a <em>Dagger</em> the 
<tt class="docutils literal"><span class="pre">Dagger.stop</span></tt> method should be called instead. See <tt class="docutils literal"><span class="pre">IMap.stop</span></tt> and
<tt class="docutils literal"><span class="pre">Dagger.stop</span></tt>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">forced(bool) [default =False]</p>
<blockquote>
<p>The <em>Piper</em> will be forced to stop the <em>IMap</em> instance. If set
<tt class="xref docutils literal"><span class="pre">True</span></tt> but the ends <em>IMap</em> argument is not specified. The 
<em>IMap</em> instance will not try to retrieve any results and will
not call the <tt class="docutils literal"><span class="pre">IMap._stop_managers</span></tt> method.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

</dd></dl>

<dl class="exception">
<dt id="papy.papy.PiperError">
<em class="property">exception </em><tt class="descclassname">papy.papy.</tt><tt class="descname">PiperError</tt><a class="headerlink" href="#papy.papy.PiperError" title="Permalink to this definition">¶</a></dt>
<dd>Exceptions raised or related to <em>Piper</em> instances.</dd></dl>

<dl class="class">
<dt id="papy.papy.Plumber">
<em class="property">class </em><tt class="descclassname">papy.papy.</tt><tt class="descname">Plumber</tt><big>(</big><em>logger_options={}</em>, <em>**kwargs</em><big>)</big><a class="headerlink" href="#papy.papy.Plumber" title="Permalink to this definition">¶</a></dt>
<dd><p>The <em>Plumber</em> is a subclass of <em>Dagger</em> and <em>Graph</em> with added run-time 
methods and a high-level interface for working with <em>PaPy</em> pipelines.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">dagger(<em>Dagger</em> instance) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
<blockquote>
<p>An optional <em>Dagger</em> instance.</p>
</blockquote>
</li>
</ul>
</blockquote>
<dl class="method">
<dt id="papy.papy.Plumber.load">
<tt class="descname">load</tt><big>(</big><em>filename</em><big>)</big><a class="headerlink" href="#papy.papy.Plumber.load" title="Permalink to this definition">¶</a></dt>
<dd><p>Load pipeline from source file.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">filename(path)</p>
<blockquote>
<p>Location of the pipeline source code.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Plumber.pause">
<tt class="descname">pause</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.papy.Plumber.pause" title="Permalink to this definition">¶</a></dt>
<dd>Pauses a running pipeline. This will stop retrieving results from the 
pipeline. Parallel parts of the pipeline will stop after the <em>IMap</em> 
buffer is has been filled. A paused pipeline can be run or stopped.</dd></dl>

<dl class="method">
<dt id="papy.papy.Plumber.run">
<tt class="descname">run</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.papy.Plumber.run" title="Permalink to this definition">¶</a></dt>
<dd>Runs a started pipeline by pulling results from it&#8217;s output <em>Pipers</em>. 
<em>Pipers</em> with the <tt class="docutils literal"><span class="pre">track</span></tt> attribute set <tt class="xref docutils literal"><span class="pre">True</span></tt> will have their 
results stored within <tt class="docutils literal"><span class="pre">Dagger.stats['pipers_tracked']</span></tt> dictionary.
A running pipeline can be paused.</dd></dl>

<dl class="method">
<dt id="papy.papy.Plumber.save">
<tt class="descname">save</tt><big>(</big><em>filename</em><big>)</big><a class="headerlink" href="#papy.papy.Plumber.save" title="Permalink to this definition">¶</a></dt>
<dd><p>Save pipeline as source file.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">filename(path)</p>
<blockquote>
<p>Path to save pipeline source code.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Plumber.start">
<tt class="descname">start</tt><big>(</big><em>datas</em><big>)</big><a class="headerlink" href="#papy.papy.Plumber.start" title="Permalink to this definition">¶</a></dt>
<dd><p>Starts the pipeline by connecting the input <em>Pipers</em> of the pipeline 
to the input data, connecting the pipeline and starting the <em>IMaps</em>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">datas(sequence)</p>
<blockquote>
<p>A sequence of external input data in the form of sequences or 
iterators. The order of items in the datas sequence should 
correspond to the order of the input <em>Pipers</em> defined by 
<tt class="docutils literal"><span class="pre">Dagger._cmp</span></tt> and <tt class="docutils literal"><span class="pre">Piper.ornament</span></tt>.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.papy.Plumber.stop">
<tt class="descname">stop</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.papy.Plumber.stop" title="Permalink to this definition">¶</a></dt>
<dd>Stops a paused pipeline. This will a trigger a <tt class="docutils literal"><span class="pre">StopIteration</span></tt> in the
inputs of the pipeline. And retrieve the buffered results. This will
stop all <em>Pipers</em> and <em>IMaps</em>. Python will not terminate cleanly if a 
pipeline is running or paused.</dd></dl>

<dl class="method">
<dt id="papy.papy.Plumber.wait">
<tt class="descname">wait</tt><big>(</big><em>timeout=None</em><big>)</big><a class="headerlink" href="#papy.papy.Plumber.wait" title="Permalink to this definition">¶</a></dt>
<dd><p>Waits(blocks) until a running pipeline finishes.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">timeout(int) [default =None]</p>
<blockquote>
<p>Specifies the timeout, <tt class="docutils literal"><span class="pre">RuntimeError</span></tt> will be raised. The 
default is to wait indefinetely for the pipeline to finish.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

</dd></dl>

<dl class="exception">
<dt id="papy.papy.PlumberError">
<em class="property">exception </em><tt class="descclassname">papy.papy.</tt><tt class="descname">PlumberError</tt><a class="headerlink" href="#papy.papy.PlumberError" title="Permalink to this definition">¶</a></dt>
<dd>Exceptions raised or related to <em>Plumber</em> instances.</dd></dl>

<dl class="class">
<dt id="papy.papy.Produce">
<em class="property">class </em><tt class="descclassname">papy.papy.</tt><tt class="descname">Produce</tt><big>(</big><em>iterable</em>, <em>n=1</em>, <em>stride=1</em><big>)</big><a class="headerlink" href="#papy.papy.Produce" title="Permalink to this definition">¶</a></dt>
<dd><p>This iterator wrapper is an iterator, but it returns elements from the 
sequence returned by the wrapped iterator. The number of returned elements
is defined by n and should not be smaller then the sequence returned by the 
wrapped iterator.</p>
<p>For example if the wrapped iterator results are ((11, 12), (21, 22), 
(31, 32)) then n <em>should</em> equal 2. For stride =1 the result will be: 
[11, 12, 21, 22, 31, 32]. For stride =2 [11, 21, 12, 22, 31, 32]. Note that
StopIteration is also an exception!</p>
</dd></dl>

<dl class="class">
<dt id="papy.papy.TeePiper">
<em class="property">class </em><tt class="descclassname">papy.papy.</tt><tt class="descname">TeePiper</tt><big>(</big><em>piper</em>, <em>i</em>, <em>stride</em><big>)</big><a class="headerlink" href="#papy.papy.TeePiper" title="Permalink to this definition">¶</a></dt>
<dd><p>This is wrapper around a <em>Piper</em>, created whenever another <em>Piper</em>
connects. The actual call to <tt class="docutils literal"><span class="pre">itertools.tee</span></tt> happens on a call to
<tt class="docutils literal"><span class="pre">Piper.start</span></tt>. A <em>TeePiper</em> protects the <tt class="docutils literal"><span class="pre">tee</span></tt> object with a threading
lock. This lock is held for a stride after this the next <em>TeePiper</em> is 
released. If a <tt class="docutils literal"><span class="pre">StopIteration</span></tt> exception occurs the next <em>TeePiper</em> is 
released and subsequent calls to the next method of this <em>TeePiper</em> will not
involve acquiring a lock and calling the <tt class="docutils literal"><span class="pre">next</span></tt> method of the wrapped 
tee object. This guarantees that the <tt class="docutils literal"><span class="pre">next</span></tt> method of a <em>Piper</em> will yield
a <tt class="docutils literal"><span class="pre">StopIteration</span></tt> only once. This guarantee is required because the <em>IMap</em>
will finish a task after the first StopIteration and will not call 
<tt class="docutils literal"><span class="pre">Piper.next</span></tt> any more and will automatically raise <tt class="docutils literal"><span class="pre">StopIterations</span></tt> for
subsequent calls to <tt class="docutils literal"><span class="pre">IMap.next</span></tt>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">piper(<em>Piper</em> instance)</p>
<blockquote>
<p>The <em>Piper</em> object to be tee&#8217;d</p>
</blockquote>
</li>
<li><p class="first">i(int)</p>
<blockquote>
<p>The index of the <tt class="docutils literal"><span class="pre">itertools.tee</span></tt> object within <tt class="docutils literal"><span class="pre">Piper.tees</span></tt>.</p>
</blockquote>
</li>
<li><p class="first">stride(int)</p>
<blockquote>
<p>The stride of the <em>Piper</em> downstream of the wrapped <em>Piper</em>. In 
a pipeline they should be the same or compatible (see manual).</p>
</blockquote>
</li>
</ul>
</blockquote>
<dl class="method">
<dt id="papy.papy.TeePiper.next">
<tt class="descname">next</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.papy.TeePiper.next" title="Permalink to this definition">¶</a></dt>
<dd>Returns or raises the next result from the <tt class="docutils literal"><span class="pre">itertools.tee</span></tt> object
for the wrapped <em>Piper</em>.</dd></dl>

</dd></dl>

<dl class="class">
<dt id="papy.papy.Worker">
<em class="property">class </em><tt class="descclassname">papy.papy.</tt><tt class="descname">Worker</tt><big>(</big><em>functions</em>, <em>arguments=None</em>, <em>kwargs=None</em>, <em>name=None</em><big>)</big><a class="headerlink" href="#papy.papy.Worker" title="Permalink to this definition">¶</a></dt>
<dd><p>The <em>Worker</em> is an object which composes sequences of functions. When called
the functions are evaluated from left to right. The function on the right 
will receive the return value from the function on the left. Optionally 
takes sequences of positional and keyworded arguments for none or
all of the composed functions. Positional arguments should be given in a 
tuple. Each element of this tuple should be a tuple of positional arguments
for the corresponding function. If a function does not take positional 
arguments its corresponding element in the arguments tuple should be an 
empty tuple i.e. <tt class="docutils literal"><span class="pre">()</span></tt>. Keyworded  arguments should be given in a tuple. 
Each  element of this tuple should be a dictionary of arguments for the 
corresponding function. If a function does not take any keyworded arguments
its corresponding element in the keyworded arguments tuple should be an 
empty dictionary i.e. <tt class="docutils literal"><span class="pre">{}</span></tt>. If none of the functions takes arguments of a 
given type the positional and/or keyworded arguments tuple can be omitted.</p>
<p>All exceptions raised by the worker-functions are caught, wrapped and 
returned <em>not</em> raised. If the <em>Worker</em> is called with a sequence which 
contains an exception no worker-function is evaluated and the exception is
wrapped and returned.</p>
<p>The <em>Worker</em> can be initialized in a variety of ways:</p>
<ul>
<li><p class="first">with a sequence of functions and a optional sequences of positional and
keyworded arguments e.g.:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">Worker</span><span class="p">((</span><span class="n">func1</span><span class="p">,</span>         <span class="n">func2</span><span class="p">,</span>    <span class="n">func3</span><span class="p">),</span> 
      <span class="p">((</span><span class="n">arg11</span><span class="p">,</span> <span class="n">arg21</span><span class="p">),</span> <span class="p">(</span><span class="n">arg21</span><span class="p">,),</span> <span class="p">()),</span>
      <span class="p">({},</span>             <span class="p">{},</span>       <span class="p">{</span><span class="s">&#39;arg31&#39;</span><span class="p">:</span><span class="n">arg31</span><span class="p">}))</span>
</pre></div>
</div>
</li>
<li><p class="first">with another <em>Worker</em> instance, which results in their functional 
equivalence e.g.:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">Worker</span><span class="p">(</span><span class="n">worker_instance</span><span class="p">)</span>
</pre></div>
</div>
</li>
<li><p class="first">With multiple <em>Worker</em> instances, where the functions and arguments of the
<em>Workers</em> are combined e.g.:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">Worker</span><span class="p">((</span><span class="n">worker1</span><span class="p">,</span> <span class="n">worker2</span><span class="p">))</span>
</pre></div>
</div>
<p>this is equivalent to:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">Worker</span><span class="p">(</span><span class="n">worker1</span><span class="o">.</span><span class="n">task</span> <span class="o">+</span> <span class="n">worker2</span><span class="o">.</span><span class="n">task</span><span class="p">,</span>                <span class="n">worker1</span><span class="o">.</span><span class="n">args</span> <span class="o">+</span> <span class="n">worker2</span><span class="o">.</span><span class="n">args</span><span class="p">,</span>                <span class="n">worker1</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">+</span> <span class="n">worker2</span><span class="o">.</span><span class="n">kwargs</span><span class="p">)</span>
</pre></div>
</div>
</li>
<li><p class="first">with a single function and its arguments in a tuple e.g.:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">Worker</span><span class="p">(</span><span class="n">function</span><span class="p">,(</span><span class="n">arg1</span><span class="p">,</span> <span class="n">arg2</span><span class="p">,</span> <span class="n">arg3</span><span class="p">))</span>
</pre></div>
</div>
<p>which is equivalent to:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">Worker</span><span class="p">((</span><span class="n">function</span><span class="p">,),((</span><span class="n">arg1</span><span class="p">,</span> <span class="n">arg2</span><span class="p">,</span> <span class="n">arg3</span><span class="p">),))</span>
</pre></div>
</div>
</li>
</ul>
</dd></dl>

<dl class="exception">
<dt id="papy.papy.WorkerError">
<em class="property">exception </em><tt class="descclassname">papy.papy.</tt><tt class="descname">WorkerError</tt><a class="headerlink" href="#papy.papy.WorkerError" title="Permalink to this definition">¶</a></dt>
<dd>Exceptions raised or related to <em>Worker</em> instances.</dd></dl>

<dl class="function">
<dt id="papy.papy.comp_task">
<tt class="descclassname">papy.papy.</tt><tt class="descname">comp_task</tt><big>(</big><em>inbox</em>, <em>args</em>, <em>kwargs</em><big>)</big><a class="headerlink" href="#papy.papy.comp_task" title="Permalink to this definition">¶</a></dt>
<dd>Composes functions in the global sequence variable TASK and evaluates the
composition given input (inbox) and arguments (args, kwargs).</dd></dl>

<dl class="function">
<dt id="papy.papy.inspect">
<tt class="descclassname">papy.papy.</tt><tt class="descname">inspect</tt><big>(</big><em>piper</em><big>)</big><a class="headerlink" href="#papy.papy.inspect" title="Permalink to this definition">¶</a></dt>
<dd>Determines the instance (Piper, Worker, FunctionType, Iterable). It returns 
a tuple of boolean variables i.e: (is_piper, is_worker, is_function, 
is_iterable_of_pipers, is_iterable_of_workers, is_iterable_of_functions).</dd></dl>

</div>
<div class="section" id="papy-graph">
<span id="module-papy.graph"></span><h1><tt class="xref docutils literal"><span class="pre">papy.graph</span></tt><a class="headerlink" href="#papy-graph" title="Permalink to this headline">¶</a></h1>
<p>This module implements a graph data structure without explicit edges, using 
nested Python dictionaries.</p>
<dl class="class">
<dt id="papy.graph.Graph">
<em class="property">class </em><tt class="descclassname">papy.graph.</tt><tt class="descname">Graph</tt><big>(</big><em>nodes=()</em>, <em>edges=()</em>, <em>xtras=None</em><big>)</big><a class="headerlink" href="#papy.graph.Graph" title="Permalink to this definition">¶</a></dt>
<dd><p>Dictionary based graph data structure.</p>
<p>This <em>Graph</em> implementation is a little bit unusual as it does not 
explicitly hold a list of edges. The <em>Graph</em> is a dictionary where the keys
of the dictionary are any hashable objects (node objects), while the values 
are <em>Node</em> instances. A <em>Node</em> instance is also a dictionary, where the keys
are node objects and the values are <em>Node</em> instances. A <em>Node</em> 
instance(value) is basically a dictionary of outgoing edges from the node 
object(key). The edges are indexed by the incoming objects. So we end up 
with a recursivly nested dictionary which defines the topology of the 
<em>Graph</em>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">nodes(sequence of nodes) [default: <tt class="docutils literal"><span class="pre">()</span></tt>]</p>
<blockquote>
<p>A sequence of nodes to be added to the graph. 
See: <tt class="docutils literal"><span class="pre">Graph.add_nodes</span></tt></p>
</blockquote>
</li>
<li><p class="first">edges(sequence of edges) [default: <tt class="docutils literal"><span class="pre">()</span></tt>]</p>
<blockquote>
<p>A sequence of edges to be added to the graph.
See: <tt class="docutils literal"><span class="pre">Graph.add_edges</span></tt></p>
</blockquote>
</li>
<li><p class="first">xtras(sequence of dictionaries) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
<blockquote>
<p>A sequence of xtra dictionaries corresponding to the added
node objects. The topological nodes corresponding to the added
dictionaries will have <tt class="docutils literal"><span class="pre">Node.xtra</span></tt> updated with the contents of 
this sequence. Either all or no xtra dictionaries have to be 
specified.</p>
</blockquote>
</li>
</ul>
</blockquote>
<dl class="method">
<dt id="papy.graph.Graph.add_edge">
<tt class="descname">add_edge</tt><big>(</big><em>edge</em>, <em>double=False</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.add_edge" title="Permalink to this definition">¶</a></dt>
<dd><p>Adds an edge to the <em>Graph</em>. An edge is just a pair of node objects. If 
the node objects are not in the <em>Graph</em> they are added.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">edge(sequence)</p>
<blockquote>
<p>An ordered pair of node objects. The edge is assumed to have a 
direction from the first to the second node object.</p>
</blockquote>
</li>
<li><p class="first">double(bool) [default: <tt class="docutils literal"><span class="pre">False`</span></tt>]</p>
<blockquote>
<p>If <tt class="xref docutils literal"><span class="pre">True</span></tt> the the reverse edge is also added.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.add_edges">
<tt class="descname">add_edges</tt><big>(</big><em>edges</em>, <em>*args</em>, <em>**kwargs</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.add_edges" title="Permalink to this definition">¶</a></dt>
<dd><p>Adds edges to the graph. Takes optional arguments for 
<tt class="docutils literal"><span class="pre">Graph.add_edge</span></tt>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">edges(sequence of edges)</p>
<blockquote>
<p>Sequence of edges to be added to the <em>Graph</em></p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.add_node">
<tt class="descname">add_node</tt><big>(</big><em>node</em>, <em>xtra=None</em>, <em>branch=None</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.add_node" title="Permalink to this definition">¶</a></dt>
<dd><p>Adds a node object to the <em>Graph</em>. Returns <tt class="xref docutils literal"><span class="pre">True</span></tt> if a new node object
has been added. If the node object is already in the <em>Graph</em> returns 
<tt class="xref docutils literal"><span class="pre">False</span></tt>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">node(object)</p>
<blockquote>
<p>Node to be added. Any hashable Python object.</p>
</blockquote>
</li>
<li><p class="first">xtra(dict) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
<blockquote>
<p>The newly created topological <tt class="docutils literal"><span class="pre">Node.xtra</span></tt> dictionary will be 
updated with the contents of this dictionary.</p>
</blockquote>
</li>
<li><p class="first">branch(object) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.add_nodes">
<tt class="descname">add_nodes</tt><big>(</big><em>nodes</em>, <em>xtras=None</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.add_nodes" title="Permalink to this definition">¶</a></dt>
<dd><p>Adds nodes to the graph.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">nodes(sequence of objects)</p>
<blockquote>
<p>Sequence of node objects to be added to the <em>Graph</em></p>
</blockquote>
</li>
<li><p class="first">xtras(sequence of dictionaries) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
<blockquote>
<p>Sequence of <tt class="docutils literal"><span class="pre">Node.xtra</span></tt> dictionaries corresponding to the node
objects being added. See: <tt class="docutils literal"><span class="pre">Graph.add_node</span></tt>.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.clear_nodes">
<tt class="descname">clear_nodes</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.graph.Graph.clear_nodes" title="Permalink to this definition">¶</a></dt>
<dd>Clears all nodes in the <em>Graph</em>. See <tt class="docutils literal"><span class="pre">Node.clear</span></tt>.</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.cmp_branch">
<tt class="descname">cmp_branch</tt><big>(</big><em>node1</em>, <em>node2</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.cmp_branch" title="Permalink to this definition">¶</a></dt>
<dd>To sort by branch of the topological Node corresponding to the node 
object.</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.deep_nodes">
<tt class="descname">deep_nodes</tt><big>(</big><em>node</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.deep_nodes" title="Permalink to this definition">¶</a></dt>
<dd>Returns all reachable node objects from a node object. See: 
<tt class="docutils literal"><span class="pre">Node.deep_nodes</span></tt></dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.del_edge">
<tt class="descname">del_edge</tt><big>(</big><em>edge</em>, <em>double=False</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.del_edge" title="Permalink to this definition">¶</a></dt>
<dd><p>Removes an edge to the <em>Graph</em>. An edges is just a pair of node objects.
But the node objects are not removed from the <em>Graph</em>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">edge(sequence)</p>
<blockquote>
<p>An ordered pair of node objects. The edge is assumed to have 
a direction from the first to the second node object.</p>
</blockquote>
</li>
<li><p class="first">double(bool) [default: <tt class="docutils literal"><span class="pre">False`</span></tt>]</p>
<blockquote>
<p>If <tt class="xref docutils literal"><span class="pre">True</span></tt> the the reverse edge is also remove.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.del_edges">
<tt class="descname">del_edges</tt><big>(</big><em>edges</em>, <em>*args</em>, <em>**kwargs</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.del_edges" title="Permalink to this definition">¶</a></dt>
<dd><p>Removes edges from the graph. Takes optional arguments for 
<tt class="docutils literal"><span class="pre">Graph.del_edge</span></tt>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">edges(sequence of edges)</p>
<blockquote>
<p>Sequence of edges to be removed from the <em>Graph</em></p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.del_node">
<tt class="descname">del_node</tt><big>(</big><em>node</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.del_node" title="Permalink to this definition">¶</a></dt>
<dd><p>Removes a node object to the <em>Graph</em>. Returns <tt class="xref docutils literal"><span class="pre">True</span></tt> if a node object
has been removed. If the node object was not in the <em>Graph</em> raises a
<tt class="docutils literal"><span class="pre">KeyError</span></tt>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">node(object)</p>
<blockquote>
<p>Node to be removed. Any hashable Python object.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.del_nodes">
<tt class="descname">del_nodes</tt><big>(</big><em>nodes</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.del_nodes" title="Permalink to this definition">¶</a></dt>
<dd><p>Removes nodes from the graph.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">nodes(sequence of objects)</p>
<blockquote>
<p>Sequence of node objects to be removed from the <em>Graph</em>. See:
<tt class="docutils literal"><span class="pre">Graph.del_node</span></tt>.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.dfs">
<tt class="descname">dfs</tt><big>(</big><em>node</em>, <em>bucket=None</em>, <em>order='append'</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.dfs" title="Permalink to this definition">¶</a></dt>
<dd><p>Recursive depth first search. By default (order = &#8216;append&#8217;) this returns
the node objects in the reverse postorder. To change this into the 
preorder use a <tt class="docutils literal"><span class="pre">collections.deque</span></tt> bucket and order &#8216;appendleft&#8217;.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">bucket(list or queue) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
<blockquote>
<p>The user <em>must</em> provide the list or queue to store the nodes</p>
</blockquote>
</li>
<li><p class="first">order(str) [default: &#8216;append&#8217;]</p>
<blockquote>
<p>Method of the bucket which will be called with the node object 
which has been examined. Other valid options might be 
&#8216;appendleft&#8217; for a <tt class="docutils literal"><span class="pre">dequeue</span></tt>.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.edges">
<tt class="descname">edges</tt><big>(</big><em>nodes=None</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.edges" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns a tuple of all edges in the <em>Graph</em>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">nodes(objects)</p>
<blockquote>
<p>If specified the edges will be limited to those originating
from one of the specified nodes.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.incoming_edges">
<tt class="descname">incoming_edges</tt><big>(</big><em>node</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.incoming_edges" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns a tuple of incoming edges for this node object.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">node(object)</p>
<blockquote>
<p>Node to be inspected for incoming edges.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.iternodes">
<tt class="descname">iternodes</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.graph.Graph.iternodes" title="Permalink to this definition">¶</a></dt>
<dd>Returns an iterator of all node objects in the <em>Graph</em></dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.node_rank">
<tt class="descname">node_rank</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.graph.Graph.node_rank" title="Permalink to this definition">¶</a></dt>
<dd>Returns the maximum rank for each node in ther graph. The rank of a node
is define as the number of edges between the node and a node which has
rank 0. A node has rank 0 if it has no incoming edges.</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.nodes">
<tt class="descname">nodes</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.graph.Graph.nodes" title="Permalink to this definition">¶</a></dt>
<dd>Returns a list of all node objects in the <em>Graph</em></dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.outgoing_edges">
<tt class="descname">outgoing_edges</tt><big>(</big><em>node</em><big>)</big><a class="headerlink" href="#papy.graph.Graph.outgoing_edges" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns a tuple of outgoing edges for this node object.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">node(object)</p>
<blockquote>
<p>Node to be inspected for outgoing edges.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.postorder">
<tt class="descname">postorder</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.graph.Graph.postorder" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns some postorder of node objects of the <em>Graph</em> if it is a 
directed acyclic graph. A postorder is not random, because the order
of elements in a dictionary is not random and so are the starting nodes 
of the depth-first search traversal which produces the postorder.
Therefore some postorders will be discovered more frequently.</p>
<p>This postorder enforces additional order:</p>
<blockquote>
<blockquote>
<ul class="simple">
<li>(TODO: describe earthworm branch order)</li>
<li>if the topological Nodes corresponding to the node objects have
a &#8216;branch&#8217; attribute it will be used to sort the graph from left 
to right.</li>
</ul>
</blockquote>
<p>is, but not unique either.</p>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="papy.graph.Graph.rank_width">
<tt class="descname">rank_width</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.graph.Graph.rank_width" title="Permalink to this definition">¶</a></dt>
<dd>Returns the width of each rank in the graph.</dd></dl>

</dd></dl>

<dl class="class">
<dt id="papy.graph.Node">
<em class="property">class </em><tt class="descclassname">papy.graph.</tt><tt class="descname">Node</tt><big>(</big><em>entity=None</em>, <em>xtra=None</em><big>)</big><a class="headerlink" href="#papy.graph.Node" title="Permalink to this definition">¶</a></dt>
<dd><p><em>Node</em> is the topological node of a <em>Graph</em>. Please note that the 
node object is not the same as the topological node. The node object is any
hashable Python object. The topological node is is defined for each node 
object and is a dictionary of other node objects with incoming edges from 
this node object.</p>
<dl class="method">
<dt id="papy.graph.Node.clear">
<tt class="descname">clear</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.graph.Node.clear" title="Permalink to this definition">¶</a></dt>
<dd>Sets the discovered and examined attributes to <tt class="xref docutils literal"><span class="pre">False</span></tt>.</dd></dl>

<dl class="method">
<dt id="papy.graph.Node.deep_nodes">
<tt class="descname">deep_nodes</tt><big>(</big><em>allnodes=None</em><big>)</big><a class="headerlink" href="#papy.graph.Node.deep_nodes" title="Permalink to this definition">¶</a></dt>
<dd>A recursive method to return <strong>all</strong> node objects reachable from this 
<em>Node</em>.</dd></dl>

<dl class="method">
<dt id="papy.graph.Node.iternodes">
<tt class="descname">iternodes</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.graph.Node.iternodes" title="Permalink to this definition">¶</a></dt>
<dd>Returns an iterator of node objects directly reachable from this <em>Node</em>.</dd></dl>

<dl class="method">
<dt id="papy.graph.Node.nodes">
<tt class="descname">nodes</tt><big>(</big><big>)</big><a class="headerlink" href="#papy.graph.Node.nodes" title="Permalink to this definition">¶</a></dt>
<dd>Returns a list of node objects directly reachable from this <em>Node</em>.</dd></dl>

</dd></dl>

</div>
<div class="section" id="papy-utils">
<span id="module-papy.utils"></span><h1><tt class="xref docutils literal"><span class="pre">papy.utils</span></tt><a class="headerlink" href="#papy-utils" title="Permalink to this headline">¶</a></h1>
<p>This module provides diverse utility functions.</p>
</div>
<div class="section" id="papy-tkgui">
<span id="module-papy.tkgui"></span><h1><tt class="xref docutils literal"><span class="pre">papy.tkgui</span></tt><a class="headerlink" href="#papy-tkgui" title="Permalink to this headline">¶</a></h1>
<p>Tkinter/Pmw gui for PaPy. Provides also a Tkinter shell widget.</p>
</div>
<div class="section" id="papy-workers">
<span id="module-papy.workers"></span><h1><tt class="xref docutils literal"><span class="pre">papy.workers</span></tt><a class="headerlink" href="#papy-workers" title="Permalink to this headline">¶</a></h1>
<p>This module provides diverse workers.</p>
<blockquote>
<ul class="simple">
<li>core (core functionality)</li>
<li>maths (wrappers around python math and operator modules)</li>
</ul>
</blockquote>
</div>
<div class="section" id="papy-workers-core">
<span id="module-papy.workers.core"></span><h1><tt class="xref docutils literal"><span class="pre">papy.workers.core</span></tt><a class="headerlink" href="#papy-workers-core" title="Permalink to this headline">¶</a></h1>
<p>A collection of core workers-functions to use in Worker instances.</p>
<dl class="function">
<dt id="papy.workers.core.ipasser">
<tt class="descclassname">papy.workers.core.</tt><tt class="descname">ipasser</tt><big>(</big><em>inbox</em>, <em>i=0</em><big>)</big><a class="headerlink" href="#papy.workers.core.ipasser" title="Permalink to this definition">¶</a></dt>
<dd><p>Passes the i-th input from inbox. By default passes the first input.</p>
<p>Arguments:</p>
<blockquote>
<ul class="simple">
<li>i(int) [default: 0]</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.core.njoiner">
<tt class="descclassname">papy.workers.core.</tt><tt class="descname">njoiner</tt><big>(</big><em>inbox</em>, <em>n=None</em>, <em>join=''</em><big>)</big><a class="headerlink" href="#papy.workers.core.njoiner" title="Permalink to this definition">¶</a></dt>
<dd><p>String joins and returns the first n inputs.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">n(int) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
<blockquote>
<p>All elements in the inbox smaller then this number will be joined.</p>
</blockquote>
</li>
<li><p class="first">join(string) [default: <tt class="docutils literal"><span class="pre">&quot;&quot;</span></tt>]</p>
<blockquote>
<p>String which will join the elements of the inbox i.e. 
<tt class="docutils literal"><span class="pre">join.join()</span></tt>.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.core.npasser">
<tt class="descclassname">papy.workers.core.</tt><tt class="descname">npasser</tt><big>(</big><em>inbox</em>, <em>n=None</em><big>)</big><a class="headerlink" href="#papy.workers.core.npasser" title="Permalink to this definition">¶</a></dt>
<dd><p>Passes n first inputs from inbox. By default passes the whole inbox.</p>
<p>Arguments:</p>
<blockquote>
<ul class="simple">
<li>n(int) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.core.nzipper">
<tt class="descclassname">papy.workers.core.</tt><tt class="descname">nzipper</tt><big>(</big><em>inbox</em>, <em>n=None</em><big>)</big><a class="headerlink" href="#papy.workers.core.nzipper" title="Permalink to this definition">¶</a></dt>
<dd><p>Zips the n first inputs from inbox. By default zips thw whole inbox.</p>
<p>Arguments:</p>
<blockquote>
<ul class="simple">
<li>n(int) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.core.plugger">
<tt class="descclassname">papy.workers.core.</tt><tt class="descname">plugger</tt><big>(</big><em>inbox</em><big>)</big><a class="headerlink" href="#papy.workers.core.plugger" title="Permalink to this definition">¶</a></dt>
<dd>Returns nothing.</dd></dl>

<dl class="function">
<dt id="papy.workers.core.sjoiner">
<tt class="descclassname">papy.workers.core.</tt><tt class="descname">sjoiner</tt><big>(</big><em>inbox</em>, <em>s=None</em>, <em>join=''</em><big>)</big><a class="headerlink" href="#papy.workers.core.sjoiner" title="Permalink to this definition">¶</a></dt>
<dd><p>String joins input with indices in s.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">s(sequence) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
<blockquote>
<p>Sequence (tuple or list) of indices of the elements which will be
joined.</p>
</blockquote>
</li>
<li><p class="first">join(string) [default: <tt class="docutils literal"><span class="pre">&quot;&quot;</span></tt>]</p>
<blockquote>
<p>String which will join the elements of the inbox i.e. 
<tt class="docutils literal"><span class="pre">join.join()</span></tt>.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.core.spasser">
<tt class="descclassname">papy.workers.core.</tt><tt class="descname">spasser</tt><big>(</big><em>inbox</em>, <em>s=None</em><big>)</big><a class="headerlink" href="#papy.workers.core.spasser" title="Permalink to this definition">¶</a></dt>
<dd><p>Passes inputs with indecies in s. By default passes the whole inbox.</p>
<p>Arguments:</p>
<blockquote>
<ul class="simple">
<li>s(sequence) [default: None -&gt; range(len(inbox))]</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.core.szipper">
<tt class="descclassname">papy.workers.core.</tt><tt class="descname">szipper</tt><big>(</big><em>inbox</em>, <em>s=None</em><big>)</big><a class="headerlink" href="#papy.workers.core.szipper" title="Permalink to this definition">¶</a></dt>
<dd><p>Zips inputs from inbox with indecies in s. By default zips thw whole inbox.</p>
<p>Arguments:</p>
<blockquote>
<ul class="simple">
<li>s(sequence) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</li>
</ul>
</blockquote>
</dd></dl>

</div>
<div class="section" id="papy-workers-maths">
<span id="module-papy.workers.maths"></span><h1><tt class="xref docutils literal"><span class="pre">papy.workers.maths</span></tt><a class="headerlink" href="#papy-workers-maths" title="Permalink to this headline">¶</a></h1>
<p>This module contains worker-functions for common math to use in <em>PaPy</em> <em>Worker</em>
instances. It wraps around functions from the <tt class="docutils literal"><span class="pre">operator</span></tt> and <tt class="docutils literal"><span class="pre">math</span></tt> Python
standard library modules. Any function from <tt class="docutils literal"><span class="pre">math</span></tt> and <tt class="docutils literal"><span class="pre">operator</span></tt> is 
included in this library if it is related to evaluations.</p>
<p>If a function accepts multiple arguments two flavours of wrappers are provided:</p>
<blockquote>
<ol class="arabic simple">
<li>simple wrapper: math.X(arg1, arg2) should be called from
papy.workers.maths.X as X([arg1], arg2)</li>
<li>star wrapper: math.X(arg1, arg2) should be called from
papy.workers.maths.Xstar ars Xstar(arg)</li>
</ol>
</blockquote>
<p>In other words 1. should be used to construct <em>Workers</em> which accept arguments 
at construction and 2. should be used to construct <em>Workers</em> which accept 
multiple inputs.</p>
</div>
<div class="section" id="papy-workers-io">
<span id="module-papy.workers.io"></span><h1><tt class="xref docutils literal"><span class="pre">papy.workers.io</span></tt><a class="headerlink" href="#papy-workers-io" title="Permalink to this headline">¶</a></h1>
<p>A collection of worker functions dealing with inputs/outputs of a pipeline or
<em>Pipers</em>. In general those functions are used to connect <em>Pipers</em> to external 
inputs/outputs (these are the pipeline input/outputs i.e. streams) or to connect
them to other <em>Pipers</em> (via items i.e. transformed elements of the input 
streams). Based on that distinction two types of functions are provided:</p>
<blockquote>
<ul class="simple">
<li>stream function - load or save the input stream from or into a single 
file, therefore they can only be used at the beginnings or ends of a 
pipeline. Stream loaders are not worker functions, as they are called once
(e.g. with the input file name as the argument) and create the input 
stream in the form of a generator of input items.</li>
<li>item functions - load, save, process or display data items. These are 
<em>Worker</em> functions and should be used within <em>Pipers</em>.</li>
</ul>
</blockquote>
<p>No method of interprocess communication, besides the default inefficient two 
pass <tt class="docutils literal"><span class="pre">multiprocessing.Queue</span></tt> and temporary files is supported on all 
platforms. Even among UNIX implementation details forking and shm implementation
details can differ.</p>
<dl class="function">
<dt id="papy.workers.io.csv_dumps">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">csv_dumps</tt><big>(</big><em>inbox</em>, <em>**kwargs</em><big>)</big><a class="headerlink" href="#papy.workers.io.csv_dumps" title="Permalink to this definition">¶</a></dt>
<dd>Dumps first element of inbox as csv (comma seperated value) string. Takes 
optional arguments for <tt class="docutils literal"><span class="pre">csv.writer</span></tt>.</dd></dl>

<dl class="function">
<dt id="papy.workers.io.csv_loads">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">csv_loads</tt><big>(</big><em>inbox</em><big>)</big><a class="headerlink" href="#papy.workers.io.csv_loads" title="Permalink to this definition">¶</a></dt>
<dd>Loads firts element of inbox as a csv (comma seperated value) string.</dd></dl>

<dl class="function">
<dt id="papy.workers.io.dump_db_item">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">dump_db_item</tt><big>(</big><em>inbox</em>, <em>type='sqlite'</em>, <em>table='temp'</em>, <em>**kwargs</em><big>)</big><a class="headerlink" href="#papy.workers.io.dump_db_item" title="Permalink to this definition">¶</a></dt>
<dd><p>Writes the first element of the inbox as a key/value pair in a database of
the provided type. Currently supported: &#8220;sqlite&#8221; and &#8220;mysql&#8221;. Returns the 
information necessary for the load_db_item to retrieve the element.</p>
<p>According to the sqlite documentation: You should avoid putting SQLite 
database files on NFS if multiple processes might try to access the file at
the same time.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">type(str) [default: &#8216;sqlite&#8217;]</p>
<blockquote>
<p>Type of the database to use currently supported &#8216;sqlite&#8217; and &#8216;mysql&#8217;
Using MySQL requires a running</p>
</blockquote>
</li>
<li><p class="first">db(str) [default: &#8216;papydb&#8217;]</p>
<blockquote>
<p>Default name of the database, for sqlite it is the name of the
database file in the current working directory. Databases can be
shared among pipers. Having multiple SQLite database files improves
concurrency. A new file will be created if none exists. The MySQL 
database has to exists (it will not be created).</p>
</blockquote>
</li>
<li><p class="first">table(str) [default: &#8216;temp&#8217;]</p>
<blockquote>
<p>Name of the table to store the key/value pairs into. Tables can be
shared among pipers.</p>
</blockquote>
</li>
<li><p class="first">host, user, passwd</p>
<blockquote>
<p>Authentication information. Refer to the generic <tt class="docutils literal"><span class="pre">dbapi2</span></tt>
documentation.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.dump_item">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">dump_item</tt><big>(</big><em>inbox</em>, <em>type='file'</em>, <em>prefix=None</em>, <em>suffix=None</em>, <em>dir=None</em>, <em>timeout=320</em>, <em>buffer=None</em><big>)</big><a class="headerlink" href="#papy.workers.io.dump_item" title="Permalink to this definition">¶</a></dt>
<dd><p>Writes the first element of the inbox as a file of a specified type. The 
type can be &#8216;file&#8217;, &#8216;fifo&#8217;, &#8216;shm&#8217;, &#8216;tcp&#8217; or &#8216;udp&#8217; corresponding to typical 
files, named pipes(FIFOs) and posix shared memory. FIFOs and shared memory
are volatile, but shared memory can exist longer then the Python process.</p>
<p>Returns the semi-random name of the file written. By default creates files 
and fifos in the default temporary directory and shared memory in /dev/shm. 
To use named pipes the operating system has to support both forks and fifos
(not Windows). To use shared memory the system has to be proper posix 
(not MacOSX) and the posix_ipc module has to be installed. Sockets should
work on all operating systems.</p>
<p>This worker-function is useful to efficently communicate parallel <em>Pipers</em>
without the overhead of using queues.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">type(&#8216;file&#8217;, &#8216;fifo&#8217;, &#8216;shm&#8217;, &#8216;tcp&#8217;, &#8216;udp&#8217;) [default: &#8216;file&#8217;]</p>
<blockquote>
<p>Type of the created file/socket.</p>
</blockquote>
</li>
<li><p class="first">prefix(str) [default: tmp_papy_%type%]</p>
<blockquote>
<p>Prefix of the file to be created. Should probably identify the
<em>Worker</em> and <em>Piper</em>.</p>
</blockquote>
</li>
<li><p class="first">suffix(str) [default: <tt class="docutils literal"><span class="pre">''</span></tt>]</p>
<blockquote>
<p>Suffix of the file to be created. Should probably identify the 
format of the serialization protocol e.g. &#8216;pickle&#8217; or
deserialized data e.g. &#8216;numpy&#8217;.</p>
</blockquote>
</li>
<li><p class="first">dir(str) [default: tempfile.gettempdir() or /dev/shm]</p>
<blockquote>
<p>Directory to safe the file to. (can be changed only for types
&#8216;file&#8217; and &#8216;fifo&#8217;</p>
</blockquote>
</li>
<li><p class="first">timeout(int) [default: 320]</p>
<blockquote>
<p>Number of seconds to keep the process at the write-end of the
socket or pipe alive.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.dump_manager_item">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">dump_manager_item</tt><big>(</big><em>inbox</em>, <em>address=('localhost'</em>, <em>46779)</em>, <em>authkey='papy'</em><big>)</big><a class="headerlink" href="#papy.workers.io.dump_manager_item" title="Permalink to this definition">¶</a></dt>
<dd><p>Writes the first element of the inbox as a shared object. The object is
stored as a value in a shared dictionary served by a <tt class="docutils literal"><span class="pre">Manager</span></tt> process.
Returns the key for the object value the address and the authentication
key.</p>
<p>To use this worker-function a <tt class="docutils literal"><span class="pre">DictServer</span></tt> instance has to be running. 
Usage of this method for IPC is not recommended for performance reasons.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">address(2-tuple) [default: (&#8216;localhost&#8217;, 46779)]</p>
<blockquote>
<p>A 2-tuple identifying the server(string) and port(integer).</p>
</blockquote>
</li>
<li><p class="first">authkey(string) [default: &#8216;papy&#8217;]</p>
<blockquote>
<p>Authentication string to connect to the server.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.dump_pickle_stream">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">dump_pickle_stream</tt><big>(</big><em>inbox</em>, <em>handle</em><big>)</big><a class="headerlink" href="#papy.workers.io.dump_pickle_stream" title="Permalink to this definition">¶</a></dt>
<dd>Writes the first element of the inbox to the provided stream (data handle) 
as a pickle. To be used with the <tt class="docutils literal"><span class="pre">load_pickle_stream</span></tt> worker-function.</dd></dl>

<dl class="function">
<dt id="papy.workers.io.dump_stream">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">dump_stream</tt><big>(</big><em>inbox</em>, <em>handle</em>, <em>delimiter=None</em><big>)</big><a class="headerlink" href="#papy.workers.io.dump_stream" title="Permalink to this definition">¶</a></dt>
<dd><p>Writes the first element of the inbox to the provided stream (file handle) 
delimiting the input by the optional delimiter string. Returns the name of
the file being written.</p>
<div class="admonition warning">
<p class="first admonition-title">Warning</p>
<p class="last">Note that only a single process can have access to a file handle open
for writing. Therefore this worker-function should only be used by a
non-parallel <em>Piper</em>.</p>
</div>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">handle(stream)</p>
<blockquote>
<p>File handle open for writing.</p>
</blockquote>
</li>
<li><p class="first">delimiter(string) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
<blockquote>
<p>A string which will seperate the written items. e.g: &#8220;END&#8221; becomes 
&#8220;nENDn&#8221; in the output stream. The default is an empty string 
which means that items will be seperated by a blank line i.e.:
&#8216;nn&#8217;</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.find_items">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">find_items</tt><big>(</big><em>prefix='tmp'</em>, <em>suffix=''</em>, <em>dir=None</em><big>)</big><a class="headerlink" href="#papy.workers.io.find_items" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a file name generator from files matching the supplied arguments. 
Matches the same files as those created by <tt class="docutils literal"><span class="pre">dump_chunk</span></tt> for the same 
arguments.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">prefix(string) [default: &#8216;tmp&#8217;]</p>
<blockquote>
<p>Mandatory first chars of the files to find.</p>
</blockquote>
</li>
<li><p class="first">suffix(string) [default: <tt class="docutils literal"><span class="pre">''</span></tt>]</p>
<blockquote>
<p>Mandatory last chars of the files to find.</p>
</blockquote>
</li>
<li><p class="first">dir(string) [default: current working directory]</p>
<blockquote>
<p>Directory where the files should be located.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.json_dumps">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">json_dumps</tt><big>(</big><em>inbox</em><big>)</big><a class="headerlink" href="#papy.workers.io.json_dumps" title="Permalink to this definition">¶</a></dt>
<dd>Serializes the first element of the input using the JSON protocol as 
implemented by the <tt class="docutils literal"><span class="pre">json</span></tt> Python 2.6 library.</dd></dl>

<dl class="function">
<dt id="papy.workers.io.json_loads">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">json_loads</tt><big>(</big><em>inbox</em><big>)</big><a class="headerlink" href="#papy.workers.io.json_loads" title="Permalink to this definition">¶</a></dt>
<dd>Deserializes the first element of the input using the JSON protocol as 
implemented by the <tt class="docutils literal"><span class="pre">json</span></tt> Python 2.6 library.</dd></dl>

<dl class="function">
<dt id="papy.workers.io.load_db_item">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">load_db_item</tt><big>(</big><em>inbox</em>, <em>remove=True</em><big>)</big><a class="headerlink" href="#papy.workers.io.load_db_item" title="Permalink to this definition">¶</a></dt>
<dd><p>Loads an item from a sqlite database. Returns the stored string.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">remove(bool) [default: <tt class="xref docutils literal"><span class="pre">True</span></tt>]</p>
<blockquote>
<p>Remove the loaded item from the table (temporary storage).</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.load_item">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">load_item</tt><big>(</big><em>inbox</em>, <em>type='string'</em>, <em>remove=True</em>, <em>buffer=None</em><big>)</big><a class="headerlink" href="#papy.workers.io.load_item" title="Permalink to this definition">¶</a></dt>
<dd><p>Loads data from a file. Determines the file type automatically (&#8216;file&#8217;,
&#8216;fifo&#8217;, &#8216;shm&#8217;, &#8216;tcp&#8217;, &#8216;udp&#8217;) but allows to specify the representation type
&#8216;string&#8217; or &#8216;mmap&#8217; for memmory mapped access to the file. Returns a the 
loaded item as a string or mmap object. Internally creates an item from a 
file object.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">type(&#8216;string&#8217; or &#8216;mmap&#8217;) [default: string]</p>
<blockquote>
<p>Determines the type of object the worker returns i.e. the file read
as a string or a memmory map. FIFOs cannot be memory mapped.</p>
</blockquote>
</li>
<li><p class="first">remove(bool) [default: <tt class="xref docutils literal"><span class="pre">True</span></tt>]</p>
<blockquote>
<p>Should the file be removed from the filesystem? This is mandatory
for FIFOs and sockets and generally a <em>very</em> good idea for shared 
memory. Files can be used to store data persistantly.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.load_manager_item">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">load_manager_item</tt><big>(</big><em>inbox</em>, <em>remove=True</em><big>)</big><a class="headerlink" href="#papy.workers.io.load_manager_item" title="Permalink to this definition">¶</a></dt>
<dd><p>Loads an item from from <tt class="docutils literal"><span class="pre">DictServer</span></tt>.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">remove(bool) [default: <tt class="xref docutils literal"><span class="pre">True</span></tt>]</p>
<blockquote>
<p>Should the data be removed from the <tt class="docutils literal"><span class="pre">DictServer</span></tt> instance?</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.load_pickle_stream">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">load_pickle_stream</tt><big>(</big><em>handle</em><big>)</big><a class="headerlink" href="#papy.workers.io.load_pickle_stream" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates an object generator from a stream (file handle) containing data
in pickles. To be used with the <tt class="docutils literal"><span class="pre">dump_pickle_stream</span></tt></p>
<div class="admonition warning">
<p class="first admonition-title">Warning</p>
<p class="last">File handles should not be read by different processes.</p>
</div>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.load_stream">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">load_stream</tt><big>(</big><em>handle</em>, <em>delimiter=None</em><big>)</big><a class="headerlink" href="#papy.workers.io.load_stream" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a string generator from a stream (file handle) containing data 
delimited by the delimiter strings. This is a stand-alone function and 
should be used to feed external data into a pipeline.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">delimiter(string) [default: <tt class="xref docutils literal"><span class="pre">None</span></tt>]</p>
<blockquote>
<p>The default means that items will be separated by two new-line 
characters i.e.: &#8216;nn&#8217;</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.make_items">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">make_items</tt><big>(</big><em>handle</em>, <em>size</em><big>)</big><a class="headerlink" href="#papy.workers.io.make_items" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a generator of items from a file handle. The size argument is the 
approximate size of the generated chunks in bytes. The main purpose of this 
worker function is to allow multiple worker processes/threads to read from 
the same file handle.</p>
<p>A chunk is a 3-tuple (file descriptor, first_byte, last_byte), which  
defines the position of chunk within a file. The size of a chunk i.e. 
last_byte - first_byte is <strong>approximately</strong> the <tt class="docutils literal"><span class="pre">size</span></tt> argument. The last
byte in a chunk is always a &#8216;n&#8217;. The first byte points 
always to the first character in a line. A chunk can also be a whole file 
i.e. the first byte is 0 and the last byte is EOF.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">size(int) [default: <tt class="docutils literal"><span class="pre">mmap.ALLOCATIONGRANULARITY</span></tt>]</p>
<blockquote>
<p>on windows: 64KBytes
on linux: 4KBytes</p>
<p>Approximate chunk size in bytes.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.make_lines">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">make_lines</tt><big>(</big><em>handle</em>, <em>follow=False</em>, <em>wait=0.10000000000000001</em><big>)</big><a class="headerlink" href="#papy.workers.io.make_lines" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a line generator from a stream (file handle) containing data in
lines.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">follow(bool) [default: <tt class="xref docutils literal"><span class="pre">False</span></tt>]</p>
<blockquote>
<p>If true follows the file after it finishes like &#8216;tail -f&#8217;.</p>
</blockquote>
</li>
<li><p class="first">wait(float) [default: 0.1]</p>
<blockquote>
<p>Time to wait between file polls.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.marshal_dumps">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">marshal_dumps</tt><big>(</big><em>inbox</em><big>)</big><a class="headerlink" href="#papy.workers.io.marshal_dumps" title="Permalink to this definition">¶</a></dt>
<dd>Serializes the first element of the input using the marshal protocol.</dd></dl>

<dl class="function">
<dt id="papy.workers.io.marshal_loads">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">marshal_loads</tt><big>(</big><em>inbox</em><big>)</big><a class="headerlink" href="#papy.workers.io.marshal_loads" title="Permalink to this definition">¶</a></dt>
<dd><p>Deserializes the first element of the input using the marshal protocol.</p>
<div class="admonition warning">
<p class="first admonition-title">Warning</p>
<p class="last">Be carful when communicating different Python version via this protocol
as it is version specific.</p>
</div>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.open_shm">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">open_shm</tt><big>(</big><em>name</em><big>)</big><a class="headerlink" href="#papy.workers.io.open_shm" title="Permalink to this definition">¶</a></dt>
<dd><p>Equivalent to the built in open function but opens a file in shared memory. 
A single file can be opened multiple times. Only the name of the file is 
necessary and not its absolute location (which is most likely 
<tt class="docutils literal"><span class="pre">/dev/shm/</span></tt>). The file is opened by default in read/write mode.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">name(string)</p>
<blockquote>
<p>The name of the file to open e.g. &#8220;my_file&#8221; not <tt class="docutils literal"><span class="pre">/dev/shm/my_file</span></tt></p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="papy.workers.io.pickle_dumps">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">pickle_dumps</tt><big>(</big><em>inbox</em><big>)</big><a class="headerlink" href="#papy.workers.io.pickle_dumps" title="Permalink to this definition">¶</a></dt>
<dd>Serializes the first element of the input using the pickle protocol using
the fastes binary protocol.</dd></dl>

<dl class="function">
<dt id="papy.workers.io.pickle_loads">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">pickle_loads</tt><big>(</big><em>inbox</em><big>)</big><a class="headerlink" href="#papy.workers.io.pickle_loads" title="Permalink to this definition">¶</a></dt>
<dd>Deserializes the first element of the input using the pickle protocol.</dd></dl>

<dl class="function">
<dt id="papy.workers.io.print_">
<tt class="descclassname">papy.workers.io.</tt><tt class="descname">print_</tt><big>(</big><em>inbox</em><big>)</big><a class="headerlink" href="#papy.workers.io.print_" title="Permalink to this definition">¶</a></dt>
<dd>Prints the first element of the inbox.</dd></dl>

</div>
<div class="section" id="imap-imap">
<span id="module-IMap.IMap"></span><h1><tt class="xref docutils literal"><span class="pre">IMap.IMap</span></tt><a class="headerlink" href="#imap-imap" title="Permalink to this headline">¶</a></h1>
<p>This module provides a parallel, buffered, multi-task, imap function. It 
evaluates results as they are needed, where the need is relative to the buffer
size. It can use threads and processes.</p>
<dl class="class">
<dt id="IMap.IMap.IMap">
<em class="property">class </em><tt class="descclassname">IMap.IMap.</tt><tt class="descname">IMap</tt><big>(</big><em>func=None</em>, <em>iterable=None</em>, <em>args=None</em>, <em>kwargs=None</em>, <em>worker_type=None</em>, <em>worker_num=None</em>, <em>worker_remote=None</em>, <em>stride=None</em>, <em>buffer=None</em>, <em>ordered=True</em>, <em>skip=False</em>, <em>name=None</em><big>)</big><a class="headerlink" href="#IMap.IMap.IMap" title="Permalink to this definition">¶</a></dt>
<dd><p>Parallel (thread- or process-based, local or remote), buffered, multi-task, 
<tt class="docutils literal"><span class="pre">itertools.imap</span></tt> or <tt class="docutils literal"><span class="pre">Pool.imap</span></tt> function replacment. Like <tt class="docutils literal"><span class="pre">imap</span></tt> it 
evaluates a function on elements of a sequence or iterator, and it does so 
lazily with an adjustable buffer. This is accomplished  via the stride and
buffer arguments. All sequences or iterators are <strong>required</strong> to be of the
same lenght. The tasks can be interdependent, the result from one task 
being the input to a second taks if the tasks are added to the <tt class="docutils literal"><span class="pre">IMap</span></tt> in 
the right order.</p>
<p>A completely lazy evaluation, i.e. submitting the first tasklet <em>after</em> the
next method for the first task has been called, is not supported.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">func, iterable, args, kwargs [default: None]</p>
<blockquote>
<p>If the IMap with those arguments they are treated to define the 
first and only task of the <tt class="docutils literal"><span class="pre">IMap</span></tt>, it <em>starts</em> the <tt class="docutils literal"><span class="pre">IMap</span></tt> pool 
and returns an <tt class="docutils literal"><span class="pre">IMap</span></tt> iterator. For a description of the args, 
kwargs and iterable input please see the add_task method. Either 
<em>both</em> or <em>none</em> func <strong>and</strong> se have to be specified. 
Positional and keyworded arguments are optional.</p>
</blockquote>
</li>
<li><p class="first">worker_type(&#8216;process&#8217; or &#8216;thread&#8217;) [default: &#8216;process&#8217;]</p>
<blockquote>
<p>Defines the type of internally spawned pool workers. For 
<tt class="docutils literal"><span class="pre">multiprocessing.Process</span></tt> based worker choose &#8216;process&#8217; for 
<tt class="docutils literal"><span class="pre">threading.Thread</span></tt> workers choose &#8216;thread&#8217;.</p>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p>This choice has <em>fundamental</em> impact on the performance of the
function. Please understand the difference between processes 
and threads and refer to the manual documentation. As a 
general rule use &#8216;process&#8217; if you have multiple CPUs or 
CPU-cores and your task functions are cpu-bound. Use &#8216;thread&#8217; 
if your function is IO-bound e.g. retrieves data from the Web.</p>
<p class="last">If you specify any remote workers via worker_remote, worker_type
has to be the default &#8216;process&#8217;. This limitation might go away 
in future versions.</p>
</div>
</blockquote>
</li>
<li><p class="first">worker_num(int) [default: number of CPUs, min: 1]</p>
<blockquote>
<p>The number of workers to spawn locally. Defaults to the number of 
availble CPUs, which is a reasonable choice for process-based  
<tt class="docutils literal"><span class="pre">IMaps</span></tt>. For thread-based <tt class="docutils literal"><span class="pre">IMaps</span></tt> a larger number of might 
improve performance. This number does <em>not</em> include workers needed
to run remote processes and can be =0 for a purely remote <em>IMap</em>.</p>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">Increasing the number of workers above the number of CPUs makes
sense only if these are Thread-based workers and the evaluated 
functions are IO-bound. Some CPU-bound tasks might evaluate 
faster if the number of worker processes equals the number of 
CPUs + 1.</p>
</div>
</blockquote>
</li>
<li><p class="first">worker_remote(sequence or None) [default: None]</p>
<blockquote>
<p>A sequence of remote host identifiers, and remote worker number 
pairs. Specifies the number of workers per <em>RPyC</em> host in the form 
of (&#8220;host:port&#8221;, worker_num). For example:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="p">[[</span><span class="s">&#39;localhost&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="s">&#39;127.0.0.1&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">]]</span>
</pre></div>
</div>
<p>with a custom TCP port:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="p">[[</span><span class="s">&#39;localhost:6666&#39;</span><span class="p">],</span> <span class="p">[</span><span class="s">&#39;remotehost:8888&#39;</span><span class="p">,</span> <span class="mi">4</span><span class="p">]]</span>
</pre></div>
</div>
</blockquote>
</li>
<li><p class="first">stride(int) [default: worker_num + sum of remote worker_num, min: 1]</p>
<blockquote>
<p>Defines the number of tasklets which are submitted to the process 
thread, consecutively from a single task. See the documentation for
<tt class="docutils literal"><span class="pre">add_task</span></tt> and the manual for an explanation of a task and 
tasklet. The stride defines the laziness of the pipeline. A long 
stride improves parallelism, but increases memory consumption. It 
should not be smaller than the size of the pool, because of idle
processes or threads.</p>
</blockquote>
</li>
<li><p class="first">buffer(int) [default: stride * (tasks * spawn), min: variable]</p>
<blockquote>
<p>The buffer argument limits the maximum memory consumption, by 
limiting the number of tasklets submitted to the pool. This number 
is larger then stride because a task might depend on results from
multiple tasks. The minimum buffer is the maximum possible number
of queued results. This number depends on the interdependencies 
between tasks, the produce/spawn/consume numbers and the stride.
The default is conservative and equals will always be enough 
regardless of the task interdependencies. Please consult the manual
before adjusting this setting.</p>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">A tasklet is considered submitted until the result is returned 
by the next method. The minimum stride is 1 this means that 
starting the IMap will cause one tasklet (the first from the 
first task) to be submitted to the pool input queue. The first 
tasklet from the second task can enter the queue only if either
the result from the first tasklet is returned or the buffer size
is larger then the stride size.
If the buffer is n then n tasklets can enter the pool. A Stride
of n requires n tasklets to enter the pool, therefore buffer 
can never be smaller then stride. If the tasks are chained i.e. 
the output from one is consumed by another then the first at 
most one tasklet i-th tasklet from each chained task is at a 
given moment in the pool. In those cases the minimum buffer to 
satisfy the worst case number of queued results is lower then 
the safe default.</p>
</div>
</blockquote>
</li>
<li><p class="first">ordered(bool) [default: True]</p>
<blockquote>
<p>If True the output of all tasks will be ordered. This means that for
specific task the result from the n-th tasklet will be return before
the result from the n+1-th tasklet. If false the results will be 
returned in the order they are calculated.</p>
</blockquote>
</li>
<li><p class="first">skip(bool) [default: False]</p>
<blockquote>
<p>Should we skip a result if trying to retrieve it raised a 
<tt class="docutils literal"><span class="pre">TimeoutError</span></tt>? If ordered =True and skip =True then the calling 
the next method after a TimeoutError will skip the result, which did
not arrive on time and try to get the next. If skip =False it will 
try to get the same result once more. If the results are not
ordered then the next result calculated will be skipped. If tasks
are chained a TimeoutError will collapse the <em>IMap</em> evaluation. Do 
<em>not</em> use specify timeouts (this argument becomes irrelevant).</p>
</blockquote>
</li>
<li><p class="first">name(string) [default: &#8216;imap_id(object)&#8217;]</p>
<blockquote>
<p>An optional name to associate with this <em>IMap</em> instance. Should be 
unique. Useful for nicer code generation.</p>
</blockquote>
</li>
</ul>
</blockquote>
<dl class="method">
<dt id="IMap.IMap.IMap.add_task">
<tt class="descname">add_task</tt><big>(</big><em>func</em>, <em>iterable</em>, <em>args=None</em>, <em>kwargs=None</em>, <em>timeout=None</em>, <em>block=True</em>, <em>track=False</em><big>)</big><a class="headerlink" href="#IMap.IMap.IMap.add_task" title="Permalink to this definition">¶</a></dt>
<dd><p>Adds a task to evaluate. A task is made of a function an iterable and optional
arguments and keyworded arguments. The iterable can be the result iterator of
a previously added task. A tasklet is a (func, iterable.next(), args, kwargs).</p>
<blockquote>
<ul>
<li><p class="first">func(callable)</p>
<blockquote>
<p>Will be called with the elements of the iterable, args and kwargs.</p>
</blockquote>
</li>
<li><p class="first">iterable(iterable)</p>
<blockquote>
<p>The elements of the iterable will be the first arguments passed to the
func.</p>
</blockquote>
</li>
<li><p class="first">args(tuple) [default =None]</p>
<blockquote>
<p>A tuple of optional constant arguments passed to the function after the
argument from the iterable.</p>
</blockquote>
</li>
<li><p class="first">kwargs(dict) [default =None]</p>
<blockquote>
<p>A dictionary of constant keyworded arguments passed to the function after
the variable argument from the iterable and the constant arguments in the
args tuple.</p>
</blockquote>
</li>
<li><p class="first">track(bool) [default =False]</p>
<blockquote>
<p>If true the results (or exceptions) of task are saved withing
self._tasks_tracked[%task_id%] as a {index:result} dictionary.
This is only useful if the task function involves creation of
persistant data. The dictionary can be used to restore the
correct order of the data.</p>
</blockquote>
</li>
</ul>
</blockquote>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">The order in which tasks are added to the IMap instance is
important. It affects the order in which tasks are submited to
the pool and consequently the order in which results should be
retrieved. If the tasks are chained then the order should be a
valid topological sort (reverse topological oreder).</p>
</div>
</dd></dl>

<dl class="method">
<dt id="IMap.IMap.IMap.get_task">
<tt class="descname">get_task</tt><big>(</big><em>task=0</em>, <em>timeout=None</em>, <em>block=True</em><big>)</big><a class="headerlink" href="#IMap.IMap.IMap.get_task" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns an iterator which results are bound to one task. The default 
iterator the one which e.g. will be used by default in for loops is the
iterator for the first task (task =0). Compare:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="k">for</span> <span class="n">result_from_task_0</span> <span class="ow">in</span> <span class="n">imap_instance</span><span class="p">:</span>
    <span class="k">pass</span>
</pre></div>
</div>
<p>with:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="k">for</span> <span class="n">result_from_task_1</span> <span class="ow">in</span> <span class="n">imap_instance</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">task_id</span> <span class="o">=</span><span class="mi">1</span><span class="p">):</span>
    <span class="k">pass</span>
</pre></div>
</div>
<p>a typical use case is:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">task_0_iterator</span> <span class="o">=</span> <span class="n">imap_instance</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">task_id</span> <span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="n">task_1_iterator</span> <span class="o">=</span> <span class="n">imap_instance</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">task_id</span> <span class="o">=</span><span class="mi">1</span><span class="p">)</span>

<span class="k">for</span> <span class="p">(</span><span class="n">task_1_res</span><span class="p">,</span> <span class="n">task_0_res</span><span class="p">)</span> <span class="ow">in</span> <span class="n">izip</span><span class="p">(</span><span class="n">task_0_iterator</span><span class="p">,</span> <span class="n">task_1_iterator</span><span class="p">):</span>
    <span class="k">pass</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="IMap.IMap.IMap.next">
<tt class="descname">next</tt><big>(</big><em>timeout=None</em>, <em>task=0</em>, <em>block=True</em><big>)</big><a class="headerlink" href="#IMap.IMap.IMap.next" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns the next result for the given task (default 0).</p>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p>the next result for task n can regardless of the buffersize be evaluated
only if the result for task n-1 has left the buffer.</p>
<p class="last">If multiple tasks are evaluated then those tasks share not only the process
or thread pool but also the buffer, the minimal buffer size is one and
therefore the results from the buffer have to be removed in the same order
as tasks are submitted to the pool. The tasks are submited in a topological
order which allows to chain them.</p>
</div>
<div class="admonition warning">
<p class="first admonition-title">Warning</p>
<p class="last">If multiple chained tasks are evaluated then the next method of only the
last should be called directly. Otherwise the pool might dead-lock depending
on the buffer-size. This is a consequenceof the fact that tasks are
submitted to the pool in a next-needed order. Calling the next method of an
up-stream task changes this topological evaluation order.</p>
</div>
</dd></dl>

<dl class="method">
<dt id="IMap.IMap.IMap.pop_task">
<tt class="descname">pop_task</tt><big>(</big><em>number</em><big>)</big><a class="headerlink" href="#IMap.IMap.IMap.pop_task" title="Permalink to this definition">¶</a></dt>
<dd><p>Removes a previously added task from the <em>IMap</em> instance.</p>
<p>Arguments</p>
<blockquote>
<ul>
<li><p class="first">number(int or True)</p>
<blockquote>
<p>A positive integer specifying the number of tasks to pop. If 
number is set <tt class="xref docutils literal"><span class="pre">True</span></tt> all tasks will be popped.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="IMap.IMap.IMap.start">
<tt class="descname">start</tt><big>(</big><em>stages=(1</em>, <em>2)</em><big>)</big><a class="headerlink" href="#IMap.IMap.IMap.start" title="Permalink to this definition">¶</a></dt>
<dd><p>Starts the processes or threads in the pool (stage 1) and the threads 
which manage the pools input and output queues respectively (stage 2).</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">stages(tuple) [default: (1,2)]</p>
<blockquote>
<p>Specifies which stages of the start process to execute.
After the first stage the pool worker processes/threads are
started and the <tt class="docutils literal"><span class="pre">IMap._started</span></tt> event is set <tt class="xref docutils literal"><span class="pre">True</span></tt>. A call
to the next method of the <em>IMap</em> instance will block. After the
second stage the <tt class="docutils literal"><span class="pre">IMap._pool_putter</span></tt> and <tt class="docutils literal"><span class="pre">IMap._pool_getter</span></tt>
threads will be started.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="method">
<dt id="IMap.IMap.IMap.stop">
<tt class="descname">stop</tt><big>(</big><em>ends=None</em>, <em>forced=False</em><big>)</big><a class="headerlink" href="#IMap.IMap.IMap.stop" title="Permalink to this definition">¶</a></dt>
<dd><p>Stops an <em>IMap</em> instance. If the list of end taks is specified via the 
ends argument it blocks the calling thread, retrieves (and discards)
a maximum of 2 * stride of results, stops the worker pool threads or 
processes and stops the threads which manage the input and output queues
of the pool respectively. If the ends argument is not specified, but
the forced argument is the method does not block and the 
<tt class="docutils literal"><span class="pre">IMap._stop_managers</span></tt> has to be called after all pending results have 
been retrieved. Either ends or forced has to be specified.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">ends(list) [default: None]</p>
<blockquote>
<p>A list of task ids which are not consumed within the IMap 
instance. All buffered results will be lost and up to 2 * stride 
of inputs consumed. If no list is given the end tasks will need 
to be manually consumed or the threads/processes might not 
terminate and the Python interpreter will not exit cleanly.</p>
</blockquote>
</li>
<li><p class="first">forced(bool) [default: False]</p>
<blockquote>
<p>If ends is not <tt class="xref docutils literal"><span class="pre">None</span></tt> this argument is ignored. If ends is 
<tt class="xref docutils literal"><span class="pre">None</span></tt> and forced is <tt class="xref docutils literal"><span class="pre">True</span></tt> the <em>IMap</em> instance will trigger
stopping mode.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="IMap.IMap.IMapTask">
<em class="property">class </em><tt class="descclassname">IMap.IMap.</tt><tt class="descname">IMapTask</tt><big>(</big><em>iterator</em>, <em>task</em>, <em>timeout</em>, <em>block</em><big>)</big><a class="headerlink" href="#IMap.IMap.IMapTask" title="Permalink to this definition">¶</a></dt>
<dd><p>The <tt class="docutils literal"><span class="pre">IMapTask</span></tt> is an object-wrapper of <tt class="docutils literal"><span class="pre">IMap</span></tt> instaces. It is an 
iterator, which returns results only for a single task. It&#8217;s next method 
does not take any arguments. <tt class="docutils literal"><span class="pre">IMap.next</span></tt> arguments are defined during
initialization.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">iterator(<tt class="docutils literal"><span class="pre">IMap</span></tt> instance)</p>
<blockquote>
<p><tt class="docutils literal"><span class="pre">IMap</span></tt> instance to wrap, usually initialization is done by the 
<tt class="docutils literal"><span class="pre">IMap.get_task</span></tt> method of the corresponding <tt class="docutils literal"><span class="pre">IMap</span></tt> instance.</p>
</blockquote>
</li>
<li><p class="first">task(integer)</p>
<blockquote>
<p>Id of the task from the <tt class="docutils literal"><span class="pre">IMap</span></tt> instance.</p>
</blockquote>
</li>
<li><p class="first">timeout</p>
<blockquote>
<p>see documentation for: <tt class="docutils literal"><span class="pre">IMap.next</span></tt>.</p>
</blockquote>
</li>
<li><p class="first">block</p>
<blockquote>
<p>see documentation for: <tt class="docutils literal"><span class="pre">IMap.next</span></tt>.</p>
</blockquote>
</li>
</ul>
</blockquote>
<dl class="method">
<dt id="IMap.IMap.IMapTask.next">
<tt class="descname">next</tt><big>(</big><big>)</big><a class="headerlink" href="#IMap.IMap.IMapTask.next" title="Permalink to this definition">¶</a></dt>
<dd>Returns a result if availble within timeout else raises a TimeoutError.
See documentation for <tt class="docutils literal"><span class="pre">IMap.next</span></tt>.</dd></dl>

</dd></dl>

<dl class="class">
<dt id="IMap.IMap.PriorityQueue">
<em class="property">class </em><tt class="descclassname">IMap.IMap.</tt><tt class="descname">PriorityQueue</tt><big>(</big><em>maxsize=0</em><big>)</big><a class="headerlink" href="#IMap.IMap.PriorityQueue" title="Permalink to this definition">¶</a></dt>
<dd>A priority queue using a heap on a list. This Queue is thread but not 
process safe.</dd></dl>

<dl class="class">
<dt id="IMap.IMap.Weave">
<em class="property">class </em><tt class="descclassname">IMap.IMap.</tt><tt class="descname">Weave</tt><big>(</big><em>iterators</em>, <em>repeats=1</em><big>)</big><a class="headerlink" href="#IMap.IMap.Weave" title="Permalink to this definition">¶</a></dt>
<dd><p>Weaves a sequence of iterators, which can be stopped if the same number of 
results has been consumed from all iterators.</p>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">iterators(sequence of iterators)</p>
<blockquote>
<p>A sequence of objects supporting the iterator protocol.</p>
</blockquote>
</li>
<li><p class="first">repeats(int) [default: 1]</p>
<blockquote>
<p>A positive integer defining the number of results return from an 
iterator in a stride i.e. before a result from the next iterator is
returned.</p>
</blockquote>
</li>
</ul>
</blockquote>
<dl class="method">
<dt id="IMap.IMap.Weave.next">
<tt class="descname">next</tt><big>(</big><big>)</big><a class="headerlink" href="#IMap.IMap.Weave.next" title="Permalink to this definition">¶</a></dt>
<dd>Returns the next element.</dd></dl>

<dl class="method">
<dt id="IMap.IMap.Weave.stop">
<tt class="descname">stop</tt><big>(</big><big>)</big><a class="headerlink" href="#IMap.IMap.Weave.stop" title="Permalink to this definition">¶</a></dt>
<dd>If called the Weave will stop at repeats boundaries.</dd></dl>

</dd></dl>

<dl class="function">
<dt id="IMap.IMap.imports">
<tt class="descclassname">IMap.IMap.</tt><tt class="descname">imports</tt><big>(</big><em>modules</em>, <em>forgive=False</em><big>)</big><a class="headerlink" href="#IMap.IMap.imports" title="Permalink to this definition">¶</a></dt>
<dd><p>Should be used as a decorator to attach import statments to function
definitions. These imports are added to the global (in Python module level) 
namespace of the decorated function.</p>
<p>Two forms of import statements are supported (in the following examples
<tt class="docutils literal"><span class="pre">foo</span></tt>, <tt class="docutils literal"><span class="pre">bar</span></tt>, <tt class="docutils literal"><span class="pre">oof,</span> <span class="pre">and</span> <span class="pre">``rab</span></tt> are modules not classes or functions):</p>
<div class="highlight-python"><div class="highlight"><pre><span class="kn">import</span> <span class="nn">foo</span><span class="o">,</span> <span class="nn">bar</span>              <span class="c"># e.q. to @imports([&#39;foo&#39;, &#39;bar&#39;])</span>
<span class="kn">import</span> <span class="nn">foo.oof</span> <span class="kn">as</span> <span class="nn">oof</span>            
<span class="kn">import</span> <span class="nn">bar.rab</span> <span class="kn">as</span> <span class="nn">rab</span>        <span class="c"># e.g. to @imports([&#39;foo.oof&#39;, &#39;bar.rab&#39;])</span>
</pre></div>
</div>
<p>Supports alternatives:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="k">try</span><span class="p">:</span>
    <span class="kn">import</span> <span class="nn">foo</span>
<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
    <span class="kn">import</span> <span class="nn">bar</span>
</pre></div>
</div>
<p>becomes:</p>
<div class="highlight-python"><pre>@imports(['foo,bar'])</pre>
</div>
<p>and:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="k">try</span><span class="p">:</span>
    <span class="kn">import</span> <span class="nn">foo.oof</span> <span class="kn">as</span> <span class="nn">oof</span>
<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
    <span class="kn">import</span> <span class="nn">bar.rab</span> <span class="kn">as</span> <span class="nn">oof</span>
</pre></div>
</div>
<p>becomes:</p>
<div class="highlight-python"><pre>@imports(['foo.oof,bar.rab'])</pre>
</div>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">This import is available in the body of the function as <tt class="docutils literal"><span class="pre">oof</span></tt></p>
</div>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">imports should be exhaustive for every decorated funcion even if two 
function have the same globals.</p>
</div>
<p>Arguments:</p>
<blockquote>
<ul>
<li><p class="first">modules(sequence)</p>
<blockquote>
<p>A list of modules in the following forms:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="p">[</span><span class="s">&#39;foo&#39;</span><span class="p">,</span> <span class="s">&#39;bar&#39;</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="s">&#39;baz&#39;</span><span class="p">]</span>
</pre></div>
</div>
<p>or:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="p">[</span><span class="s">&#39;foo.oof&#39;</span><span class="p">,</span> <span class="s">&#39;bar.rab&#39;</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="s">&#39;baz.zab&#39;</span><span class="p">]</span>
</pre></div>
</div>
</blockquote>
</li>
<li><p class="first">forgive(bool) [default: <tt class="xref docutils literal"><span class="pre">False</span></tt>]</p>
<blockquote>
<p>If <tt class="xref docutils literal"><span class="pre">True</span></tt> will not raise exception on <tt class="docutils literal"><span class="pre">ImportError</span></tt>.</p>
</blockquote>
</li>
</ul>
</blockquote>
</dd></dl>

<dl class="function">
<dt id="IMap.IMap.inject_func">
<tt class="descclassname">IMap.IMap.</tt><tt class="descname">inject_func</tt><big>(</big><em>func</em>, <em>conn</em><big>)</big><a class="headerlink" href="#IMap.IMap.inject_func" title="Permalink to this definition">¶</a></dt>
<dd>Injects a function object into a rpyc connection object.</dd></dl>

<dl class="function">
<dt id="IMap.IMap.worker">
<tt class="descclassname">IMap.IMap.</tt><tt class="descname">worker</tt><big>(</big><em>inqueue</em>, <em>outqueue</em>, <em>host=None</em><big>)</big><a class="headerlink" href="#IMap.IMap.worker" title="Permalink to this definition">¶</a></dt>
<dd>Function which is executed by processes/threads within the pool.
It waits for tasks (function, data, arguments) at the input queue
evaluates the result and passes it to the output queue.</dd></dl>

</div>
<p># EOF</p>


   
 
 
 </div>
<div id="footer" dir="ltr">
 
 <div class="text">
 
 &copy;2008 Google -
 <a href="http://code.google.com/">Code Home</a> -
 <a href="http://code.google.com/tos.html">Terms of Service</a> -
 <a href="http://www.google.com/privacy.html">Privacy Policy</a> -
  
 </body>
</html>
